2024-11-13 18:27:04,081 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-13 18:27:04,120 main DEBUG Took 0.036491 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 18:27:04,121 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 18:27:04,121 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 18:27:04,122 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 18:27:04,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,135 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 18:27:04,148 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,151 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,152 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,154 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,155 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,156 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,158 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,158 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,159 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,160 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,161 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,162 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,162 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,163 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,163 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,164 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:27:04,165 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,165 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 18:27:04,167 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:27:04,169 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 18:27:04,171 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 18:27:04,172 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 18:27:04,174 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 18:27:04,174 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 18:27:04,185 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 18:27:04,188 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 18:27:04,191 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 18:27:04,191 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 18:27:04,191 main DEBUG createAppenders(={Console}) 2024-11-13 18:27:04,192 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-13 18:27:04,193 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-13 18:27:04,193 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-13 18:27:04,194 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 18:27:04,194 main DEBUG OutputStream closed 2024-11-13 18:27:04,195 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 18:27:04,195 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 18:27:04,195 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-13 18:27:04,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 18:27:04,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 18:27:04,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 18:27:04,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 18:27:04,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 18:27:04,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 18:27:04,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 18:27:04,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 18:27:04,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 18:27:04,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 18:27:04,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 18:27:04,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 18:27:04,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 18:27:04,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 18:27:04,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 18:27:04,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 18:27:04,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 18:27:04,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 18:27:04,298 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 18:27:04,299 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-13 18:27:04,299 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 18:27:04,300 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-13T18:27:04,314 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-13 18:27:04,318 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 18:27:04,318 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T18:27:04,387 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=500, ProcessCount=11, AvailableMemoryMB=3123 2024-11-13T18:27:04,722 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c 2024-11-13T18:27:05,471 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T18:27:05,578 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T18:27:05,645 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-13T18:27:05,704 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522425692 2024-11-13T18:27:05,793 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522425692 2024-11-13T18:27:05,890 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:05,897 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-13T18:27:05,897 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-13T18:27:05,938 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 47817b19762749c530148ffa4f05a124, NAME => 'testLockupAroundBadAssignSync,,1731522425912.47817b19762749c530148ffa4f05a124.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c 2024-11-13T18:27:05,991 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T18:27:05,997 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1731522425912.47817b19762749c530148ffa4f05a124.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:27:06,123 INFO [StoreOpener-47817b19762749c530148ffa4f05a124-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region 47817b19762749c530148ffa4f05a124 2024-11-13T18:27:06,179 INFO [StoreOpener-47817b19762749c530148ffa4f05a124-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47817b19762749c530148ffa4f05a124 columnFamilyName MyCF 2024-11-13T18:27:06,188 DEBUG [StoreOpener-47817b19762749c530148ffa4f05a124-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:27:06,196 INFO [StoreOpener-47817b19762749c530148ffa4f05a124-1 {}] regionserver.HStore(327): Store=47817b19762749c530148ffa4f05a124/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:27:06,199 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47817b19762749c530148ffa4f05a124 2024-11-13T18:27:06,205 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/data/default/testLockupAroundBadAssignSync/47817b19762749c530148ffa4f05a124 2024-11-13T18:27:06,205 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/data/default/testLockupAroundBadAssignSync/47817b19762749c530148ffa4f05a124 2024-11-13T18:27:06,208 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47817b19762749c530148ffa4f05a124 2024-11-13T18:27:06,209 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47817b19762749c530148ffa4f05a124 2024-11-13T18:27:06,217 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47817b19762749c530148ffa4f05a124 2024-11-13T18:27:06,239 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/data/default/testLockupAroundBadAssignSync/47817b19762749c530148ffa4f05a124/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:27:06,241 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47817b19762749c530148ffa4f05a124; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73155842, jitterRate=0.09010699391365051}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:27:06,251 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T18:27:06,271 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47817b19762749c530148ffa4f05a124: Writing region info on filesystem at 1731522426067Initializing all the Stores at 1731522426068 (+1 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1731522426069 (+1 ms)Cleaning up temporary data from old regions at 1731522426209 (+140 ms)Region opened successfully at 1731522426267 (+58 ms) 2024-11-13T18:27:06,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522425692) roll requested 2024-11-13T18:27:06,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426337 2024-11-13T18:27:06,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522425692 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426337 2024-11-13T18:27:06,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426337) roll requested 2024-11-13T18:27:06,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426353 2024-11-13T18:27:06,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426353 2024-11-13T18:27:06,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426353) roll requested 2024-11-13T18:27:06,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426374 2024-11-13T18:27:06,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426374 2024-11-13T18:27:06,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426374) roll requested 2024-11-13T18:27:06,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426400 2024-11-13T18:27:06,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426400 2024-11-13T18:27:06,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426400) roll requested 2024-11-13T18:27:06,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426423 2024-11-13T18:27:06,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426423 2024-11-13T18:27:06,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426423) roll requested 2024-11-13T18:27:06,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426453 2024-11-13T18:27:06,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426453 2024-11-13T18:27:06,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426453) roll requested 2024-11-13T18:27:06,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426491 2024-11-13T18:27:06,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426491 2024-11-13T18:27:06,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426491) roll requested 2024-11-13T18:27:06,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426532 2024-11-13T18:27:06,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426532 2024-11-13T18:27:06,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426532) roll requested 2024-11-13T18:27:06,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426562 2024-11-13T18:27:06,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426562 2024-11-13T18:27:06,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426562) roll requested 2024-11-13T18:27:06,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426599 2024-11-13T18:27:06,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426599 2024-11-13T18:27:06,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426599) roll requested 2024-11-13T18:27:06,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426618 2024-11-13T18:27:06,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426618 2024-11-13T18:27:06,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426618) roll requested 2024-11-13T18:27:06,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426630 2024-11-13T18:27:06,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426630 2024-11-13T18:27:06,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426630) roll requested 2024-11-13T18:27:06,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426646 2024-11-13T18:27:06,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426646 2024-11-13T18:27:06,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426646) roll requested 2024-11-13T18:27:06,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426661 2024-11-13T18:27:06,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426661 2024-11-13T18:27:06,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426661) roll requested 2024-11-13T18:27:06,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426678 2024-11-13T18:27:06,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426678 2024-11-13T18:27:06,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426678) roll requested 2024-11-13T18:27:06,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426693 2024-11-13T18:27:06,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426693 2024-11-13T18:27:06,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426693) roll requested 2024-11-13T18:27:06,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426708 2024-11-13T18:27:06,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426708 2024-11-13T18:27:06,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426708) roll requested 2024-11-13T18:27:06,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426724 2024-11-13T18:27:06,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426724 2024-11-13T18:27:06,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426724) roll requested 2024-11-13T18:27:06,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426743 2024-11-13T18:27:06,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426743 2024-11-13T18:27:06,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426743) roll requested 2024-11-13T18:27:06,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426759 2024-11-13T18:27:06,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426759 2024-11-13T18:27:06,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426759) roll requested 2024-11-13T18:27:06,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426776 2024-11-13T18:27:06,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426776 2024-11-13T18:27:06,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426776) roll requested 2024-11-13T18:27:06,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426795 2024-11-13T18:27:06,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426795 2024-11-13T18:27:06,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426795) roll requested 2024-11-13T18:27:06,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426820 2024-11-13T18:27:06,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426820 2024-11-13T18:27:06,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426820) roll requested 2024-11-13T18:27:06,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426838 2024-11-13T18:27:06,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426838 2024-11-13T18:27:06,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426838) roll requested 2024-11-13T18:27:06,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426854 2024-11-13T18:27:06,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426854 2024-11-13T18:27:06,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426854) roll requested 2024-11-13T18:27:06,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426868 2024-11-13T18:27:06,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426868 2024-11-13T18:27:06,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426868) roll requested 2024-11-13T18:27:06,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426883 2024-11-13T18:27:06,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426883 2024-11-13T18:27:06,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426883) roll requested 2024-11-13T18:27:06,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426906 2024-11-13T18:27:06,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426906 2024-11-13T18:27:06,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426906) roll requested 2024-11-13T18:27:06,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426946 2024-11-13T18:27:06,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426946 2024-11-13T18:27:06,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426946) roll requested 2024-11-13T18:27:06,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426978 2024-11-13T18:27:06,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:06,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426978 2024-11-13T18:27:06,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:06,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:06,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426978) roll requested 2024-11-13T18:27:06,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522426994 2024-11-13T18:27:07,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426994 2024-11-13T18:27:07,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522426994) roll requested 2024-11-13T18:27:07,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427014 2024-11-13T18:27:07,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522426994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427014 2024-11-13T18:27:07,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427014) roll requested 2024-11-13T18:27:07,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427026 2024-11-13T18:27:07,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427026 2024-11-13T18:27:07,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427026) roll requested 2024-11-13T18:27:07,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427037 2024-11-13T18:27:07,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427037 2024-11-13T18:27:07,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427037) roll requested 2024-11-13T18:27:07,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427046 2024-11-13T18:27:07,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427046 2024-11-13T18:27:07,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427046) roll requested 2024-11-13T18:27:07,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427055 2024-11-13T18:27:07,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427055 2024-11-13T18:27:07,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427055) roll requested 2024-11-13T18:27:07,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427065 2024-11-13T18:27:07,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427065 2024-11-13T18:27:07,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427065) roll requested 2024-11-13T18:27:07,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427077 2024-11-13T18:27:07,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427077 2024-11-13T18:27:07,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427077) roll requested 2024-11-13T18:27:07,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427088 2024-11-13T18:27:07,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427088 2024-11-13T18:27:07,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427088) roll requested 2024-11-13T18:27:07,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427100 2024-11-13T18:27:07,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427100 2024-11-13T18:27:07,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427100) roll requested 2024-11-13T18:27:07,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427110 2024-11-13T18:27:07,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427110 2024-11-13T18:27:07,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427110) roll requested 2024-11-13T18:27:07,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427124 2024-11-13T18:27:07,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427124 2024-11-13T18:27:07,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427124) roll requested 2024-11-13T18:27:07,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427140 2024-11-13T18:27:07,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427140 2024-11-13T18:27:07,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427140) roll requested 2024-11-13T18:27:07,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427158 2024-11-13T18:27:07,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427158 2024-11-13T18:27:07,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427158) roll requested 2024-11-13T18:27:07,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427169 2024-11-13T18:27:07,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427169 2024-11-13T18:27:07,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427169) roll requested 2024-11-13T18:27:07,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427178 2024-11-13T18:27:07,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427178 2024-11-13T18:27:07,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427178) roll requested 2024-11-13T18:27:07,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427187 2024-11-13T18:27:07,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427187 2024-11-13T18:27:07,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427187) roll requested 2024-11-13T18:27:07,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427198 2024-11-13T18:27:07,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427198 2024-11-13T18:27:07,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427198) roll requested 2024-11-13T18:27:07,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427207 2024-11-13T18:27:07,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427207 2024-11-13T18:27:07,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427207) roll requested 2024-11-13T18:27:07,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427220 2024-11-13T18:27:07,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427220 2024-11-13T18:27:07,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427220) roll requested 2024-11-13T18:27:07,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427229 2024-11-13T18:27:07,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427229 2024-11-13T18:27:07,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427229) roll requested 2024-11-13T18:27:07,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427241 2024-11-13T18:27:07,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427241 2024-11-13T18:27:07,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427241) roll requested 2024-11-13T18:27:07,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427251 2024-11-13T18:27:07,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427251 2024-11-13T18:27:07,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427251) roll requested 2024-11-13T18:27:07,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427269 2024-11-13T18:27:07,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427269 2024-11-13T18:27:07,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427269) roll requested 2024-11-13T18:27:07,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427278 2024-11-13T18:27:07,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427278 2024-11-13T18:27:07,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427278) roll requested 2024-11-13T18:27:07,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427291 2024-11-13T18:27:07,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427291 2024-11-13T18:27:07,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427291) roll requested 2024-11-13T18:27:07,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427305 2024-11-13T18:27:07,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427305 2024-11-13T18:27:07,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427305) roll requested 2024-11-13T18:27:07,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427324 2024-11-13T18:27:07,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427324 2024-11-13T18:27:07,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427324) roll requested 2024-11-13T18:27:07,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427355 2024-11-13T18:27:07,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427355 2024-11-13T18:27:07,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427355) roll requested 2024-11-13T18:27:07,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427384 2024-11-13T18:27:07,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427384 2024-11-13T18:27:07,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427384) roll requested 2024-11-13T18:27:07,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427412 2024-11-13T18:27:07,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427412 2024-11-13T18:27:07,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427412) roll requested 2024-11-13T18:27:07,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427441 2024-11-13T18:27:07,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427441 2024-11-13T18:27:07,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427441) roll requested 2024-11-13T18:27:07,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427473 2024-11-13T18:27:07,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427473 2024-11-13T18:27:07,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427473) roll requested 2024-11-13T18:27:07,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427511 2024-11-13T18:27:07,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427511 2024-11-13T18:27:07,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427511) roll requested 2024-11-13T18:27:07,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427550 2024-11-13T18:27:07,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427550 2024-11-13T18:27:07,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427550) roll requested 2024-11-13T18:27:07,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427604 2024-11-13T18:27:07,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427604 2024-11-13T18:27:07,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427604) roll requested 2024-11-13T18:27:07,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427647 2024-11-13T18:27:07,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427647 2024-11-13T18:27:07,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427647) roll requested 2024-11-13T18:27:07,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427693 2024-11-13T18:27:07,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427693 2024-11-13T18:27:07,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427693) roll requested 2024-11-13T18:27:07,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427767 2024-11-13T18:27:07,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427767 2024-11-13T18:27:07,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427767) roll requested 2024-11-13T18:27:07,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427799 2024-11-13T18:27:07,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427799 2024-11-13T18:27:07,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427799) roll requested 2024-11-13T18:27:07,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427834 2024-11-13T18:27:07,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427834 2024-11-13T18:27:07,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427834) roll requested 2024-11-13T18:27:07,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427871 2024-11-13T18:27:07,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427871 2024-11-13T18:27:07,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427871) roll requested 2024-11-13T18:27:07,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427880 2024-11-13T18:27:07,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427880 2024-11-13T18:27:07,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427880) roll requested 2024-11-13T18:27:07,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427891 2024-11-13T18:27:07,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427891 2024-11-13T18:27:07,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427891) roll requested 2024-11-13T18:27:07,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427900 2024-11-13T18:27:07,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427900 2024-11-13T18:27:07,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427900) roll requested 2024-11-13T18:27:07,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427909 2024-11-13T18:27:07,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427909 2024-11-13T18:27:07,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427909) roll requested 2024-11-13T18:27:07,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427922 2024-11-13T18:27:07,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427922 2024-11-13T18:27:07,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427922) roll requested 2024-11-13T18:27:07,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427932 2024-11-13T18:27:07,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427932 2024-11-13T18:27:07,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427932) roll requested 2024-11-13T18:27:07,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427944 2024-11-13T18:27:07,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427944 2024-11-13T18:27:07,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427944) roll requested 2024-11-13T18:27:07,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427955 2024-11-13T18:27:07,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427955 2024-11-13T18:27:07,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427955) roll requested 2024-11-13T18:27:07,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427977 2024-11-13T18:27:07,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:07,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427977 2024-11-13T18:27:07,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:07,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:07,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427977) roll requested 2024-11-13T18:27:07,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522427993 2024-11-13T18:27:08,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427993 2024-11-13T18:27:08,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522427993) roll requested 2024-11-13T18:27:08,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428009 2024-11-13T18:27:08,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522427993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428009 2024-11-13T18:27:08,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428009) roll requested 2024-11-13T18:27:08,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428021 2024-11-13T18:27:08,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428021 2024-11-13T18:27:08,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428021) roll requested 2024-11-13T18:27:08,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428033 2024-11-13T18:27:08,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428033 2024-11-13T18:27:08,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428033) roll requested 2024-11-13T18:27:08,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428045 2024-11-13T18:27:08,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428045 2024-11-13T18:27:08,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428045) roll requested 2024-11-13T18:27:08,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428067 2024-11-13T18:27:08,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428067 2024-11-13T18:27:08,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428067) roll requested 2024-11-13T18:27:08,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428080 2024-11-13T18:27:08,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428080 2024-11-13T18:27:08,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428080) roll requested 2024-11-13T18:27:08,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428117 2024-11-13T18:27:08,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428117 2024-11-13T18:27:08,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428117) roll requested 2024-11-13T18:27:08,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428140 2024-11-13T18:27:08,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428140 2024-11-13T18:27:08,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428140) roll requested 2024-11-13T18:27:08,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428169 2024-11-13T18:27:08,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428169 2024-11-13T18:27:08,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428169) roll requested 2024-11-13T18:27:08,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428202 2024-11-13T18:27:08,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428202 2024-11-13T18:27:08,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428202) roll requested 2024-11-13T18:27:08,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428258 2024-11-13T18:27:08,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428258 2024-11-13T18:27:08,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428258) roll requested 2024-11-13T18:27:08,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428273 2024-11-13T18:27:08,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428273 2024-11-13T18:27:08,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428273) roll requested 2024-11-13T18:27:08,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428286 2024-11-13T18:27:08,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428286 2024-11-13T18:27:08,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428286) roll requested 2024-11-13T18:27:08,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428296 2024-11-13T18:27:08,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428296 2024-11-13T18:27:08,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428296) roll requested 2024-11-13T18:27:08,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428307 2024-11-13T18:27:08,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428307 2024-11-13T18:27:08,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428307) roll requested 2024-11-13T18:27:08,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428316 2024-11-13T18:27:08,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428316 2024-11-13T18:27:08,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428316) roll requested 2024-11-13T18:27:08,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428326 2024-11-13T18:27:08,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428326 2024-11-13T18:27:08,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428326) roll requested 2024-11-13T18:27:08,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428335 2024-11-13T18:27:08,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428335 2024-11-13T18:27:08,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428335) roll requested 2024-11-13T18:27:08,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428343 2024-11-13T18:27:08,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428343 2024-11-13T18:27:08,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428343) roll requested 2024-11-13T18:27:08,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428350 2024-11-13T18:27:08,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428350 2024-11-13T18:27:08,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428350) roll requested 2024-11-13T18:27:08,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428357 2024-11-13T18:27:08,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428357 2024-11-13T18:27:08,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428350 is not closed yet, will try archiving it next time 2024-11-13T18:27:08,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428357) roll requested 2024-11-13T18:27:08,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428365 2024-11-13T18:27:08,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428365 2024-11-13T18:27:08,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428365) roll requested 2024-11-13T18:27:08,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428374 2024-11-13T18:27:08,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428374 2024-11-13T18:27:08,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428374) roll requested 2024-11-13T18:27:08,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428381 2024-11-13T18:27:08,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428381 2024-11-13T18:27:08,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428381) roll requested 2024-11-13T18:27:08,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428392 2024-11-13T18:27:08,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428392 2024-11-13T18:27:08,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428392) roll requested 2024-11-13T18:27:08,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428401 2024-11-13T18:27:08,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428401 2024-11-13T18:27:08,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428401) roll requested 2024-11-13T18:27:08,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428410 2024-11-13T18:27:08,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428410 2024-11-13T18:27:08,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428410) roll requested 2024-11-13T18:27:08,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428424 2024-11-13T18:27:08,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428424 2024-11-13T18:27:08,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428424) roll requested 2024-11-13T18:27:08,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428433 2024-11-13T18:27:08,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428433 2024-11-13T18:27:08,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428433) roll requested 2024-11-13T18:27:08,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428444 2024-11-13T18:27:08,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428444 2024-11-13T18:27:08,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428444) roll requested 2024-11-13T18:27:08,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428452 2024-11-13T18:27:08,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428452 2024-11-13T18:27:08,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428452) roll requested 2024-11-13T18:27:08,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428461 2024-11-13T18:27:08,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428461 2024-11-13T18:27:08,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428461) roll requested 2024-11-13T18:27:08,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428472 2024-11-13T18:27:08,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428472 2024-11-13T18:27:08,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428472) roll requested 2024-11-13T18:27:08,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428481 2024-11-13T18:27:08,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428481 2024-11-13T18:27:08,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428481) roll requested 2024-11-13T18:27:08,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428491 2024-11-13T18:27:08,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428491 2024-11-13T18:27:08,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428491) roll requested 2024-11-13T18:27:08,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428504 2024-11-13T18:27:08,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428504 2024-11-13T18:27:08,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428504) roll requested 2024-11-13T18:27:08,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428528 2024-11-13T18:27:08,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428528 2024-11-13T18:27:08,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428528) roll requested 2024-11-13T18:27:08,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428545 2024-11-13T18:27:08,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428545 2024-11-13T18:27:08,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428545) roll requested 2024-11-13T18:27:08,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428556 2024-11-13T18:27:08,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428556 2024-11-13T18:27:08,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428556) roll requested 2024-11-13T18:27:08,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428568 2024-11-13T18:27:08,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428568 2024-11-13T18:27:08,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428568) roll requested 2024-11-13T18:27:08,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428576 2024-11-13T18:27:08,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428576 2024-11-13T18:27:08,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428576) roll requested 2024-11-13T18:27:08,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428585 2024-11-13T18:27:08,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428585 2024-11-13T18:27:08,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428585) roll requested 2024-11-13T18:27:08,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428593 2024-11-13T18:27:08,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428593 2024-11-13T18:27:08,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428593) roll requested 2024-11-13T18:27:08,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428604 2024-11-13T18:27:08,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428604 2024-11-13T18:27:08,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428604) roll requested 2024-11-13T18:27:08,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428612 2024-11-13T18:27:08,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428612 2024-11-13T18:27:08,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428612) roll requested 2024-11-13T18:27:08,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428621 2024-11-13T18:27:08,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428621 2024-11-13T18:27:08,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428621) roll requested 2024-11-13T18:27:08,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428628 2024-11-13T18:27:08,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428628 2024-11-13T18:27:08,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428628) roll requested 2024-11-13T18:27:08,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428637 2024-11-13T18:27:08,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428637 2024-11-13T18:27:08,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428637) roll requested 2024-11-13T18:27:08,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428653 2024-11-13T18:27:08,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428653 2024-11-13T18:27:08,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428653) roll requested 2024-11-13T18:27:08,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428662 2024-11-13T18:27:08,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428662 2024-11-13T18:27:08,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428662) roll requested 2024-11-13T18:27:08,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428676 2024-11-13T18:27:08,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428676 2024-11-13T18:27:08,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428676) roll requested 2024-11-13T18:27:08,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428702 2024-11-13T18:27:08,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428702 2024-11-13T18:27:08,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428702) roll requested 2024-11-13T18:27:08,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428742 2024-11-13T18:27:08,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428742 2024-11-13T18:27:08,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428742) roll requested 2024-11-13T18:27:08,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428775 2024-11-13T18:27:08,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428775 2024-11-13T18:27:08,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428775) roll requested 2024-11-13T18:27:08,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428818 2024-11-13T18:27:08,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428818 2024-11-13T18:27:08,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428818) roll requested 2024-11-13T18:27:08,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428853 2024-11-13T18:27:08,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428853 2024-11-13T18:27:08,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428853) roll requested 2024-11-13T18:27:08,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428875 2024-11-13T18:27:08,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428875 2024-11-13T18:27:08,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428875) roll requested 2024-11-13T18:27:08,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428887 2024-11-13T18:27:08,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428887 2024-11-13T18:27:08,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428887) roll requested 2024-11-13T18:27:08,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428896 2024-11-13T18:27:08,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428896 2024-11-13T18:27:08,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428896) roll requested 2024-11-13T18:27:08,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428911 2024-11-13T18:27:08,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428911 2024-11-13T18:27:08,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428911) roll requested 2024-11-13T18:27:08,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428922 2024-11-13T18:27:08,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428922 2024-11-13T18:27:08,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428922) roll requested 2024-11-13T18:27:08,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428933 2024-11-13T18:27:08,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428933 2024-11-13T18:27:08,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428933) roll requested 2024-11-13T18:27:08,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428949 2024-11-13T18:27:08,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428949 2024-11-13T18:27:08,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428949) roll requested 2024-11-13T18:27:08,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428961 2024-11-13T18:27:08,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428961 2024-11-13T18:27:08,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428961) roll requested 2024-11-13T18:27:08,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428969 2024-11-13T18:27:08,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428969 2024-11-13T18:27:08,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428969) roll requested 2024-11-13T18:27:08,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428986 2024-11-13T18:27:08,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:08,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428986 2024-11-13T18:27:08,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:08,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:08,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428986) roll requested 2024-11-13T18:27:08,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522428999 2024-11-13T18:27:09,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428999 2024-11-13T18:27:09,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522428999) roll requested 2024-11-13T18:27:09,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429011 2024-11-13T18:27:09,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522428999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429011 2024-11-13T18:27:09,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429011) roll requested 2024-11-13T18:27:09,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429025 2024-11-13T18:27:09,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429025 2024-11-13T18:27:09,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429025) roll requested 2024-11-13T18:27:09,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429039 2024-11-13T18:27:09,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429039 2024-11-13T18:27:09,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429039) roll requested 2024-11-13T18:27:09,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429053 2024-11-13T18:27:09,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429053 2024-11-13T18:27:09,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429053) roll requested 2024-11-13T18:27:09,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429069 2024-11-13T18:27:09,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429069 2024-11-13T18:27:09,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429069) roll requested 2024-11-13T18:27:09,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429085 2024-11-13T18:27:09,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429085 2024-11-13T18:27:09,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429085) roll requested 2024-11-13T18:27:09,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429101 2024-11-13T18:27:09,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429101 2024-11-13T18:27:09,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429101) roll requested 2024-11-13T18:27:09,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429124 2024-11-13T18:27:09,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429124 2024-11-13T18:27:09,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429124) roll requested 2024-11-13T18:27:09,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429133 2024-11-13T18:27:09,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429133 2024-11-13T18:27:09,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429133) roll requested 2024-11-13T18:27:09,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429145 2024-11-13T18:27:09,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429145 2024-11-13T18:27:09,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429145) roll requested 2024-11-13T18:27:09,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429153 2024-11-13T18:27:09,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429153 2024-11-13T18:27:09,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429153) roll requested 2024-11-13T18:27:09,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429166 2024-11-13T18:27:09,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429166 2024-11-13T18:27:09,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429166) roll requested 2024-11-13T18:27:09,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429177 2024-11-13T18:27:09,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429177 2024-11-13T18:27:09,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429177) roll requested 2024-11-13T18:27:09,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429186 2024-11-13T18:27:09,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429186 2024-11-13T18:27:09,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429186) roll requested 2024-11-13T18:27:09,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429198 2024-11-13T18:27:09,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429198 2024-11-13T18:27:09,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429198) roll requested 2024-11-13T18:27:09,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429206 2024-11-13T18:27:09,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429206 2024-11-13T18:27:09,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429206) roll requested 2024-11-13T18:27:09,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429214 2024-11-13T18:27:09,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429214 2024-11-13T18:27:09,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429214) roll requested 2024-11-13T18:27:09,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429225 2024-11-13T18:27:09,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429225 2024-11-13T18:27:09,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429225) roll requested 2024-11-13T18:27:09,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429235 2024-11-13T18:27:09,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429235 2024-11-13T18:27:09,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429235) roll requested 2024-11-13T18:27:09,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429242 2024-11-13T18:27:09,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429242 2024-11-13T18:27:09,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429242) roll requested 2024-11-13T18:27:09,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429251 2024-11-13T18:27:09,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429251 2024-11-13T18:27:09,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429251) roll requested 2024-11-13T18:27:09,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429259 2024-11-13T18:27:09,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429259 2024-11-13T18:27:09,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429259) roll requested 2024-11-13T18:27:09,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429267 2024-11-13T18:27:09,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429267 2024-11-13T18:27:09,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429267) roll requested 2024-11-13T18:27:09,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429275 2024-11-13T18:27:09,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429275 2024-11-13T18:27:09,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429275) roll requested 2024-11-13T18:27:09,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429305 2024-11-13T18:27:09,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429305 2024-11-13T18:27:09,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429305) roll requested 2024-11-13T18:27:09,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429326 2024-11-13T18:27:09,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429326 2024-11-13T18:27:09,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429326) roll requested 2024-11-13T18:27:09,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429341 2024-11-13T18:27:09,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429341 2024-11-13T18:27:09,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429341) roll requested 2024-11-13T18:27:09,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429358 2024-11-13T18:27:09,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429358 2024-11-13T18:27:09,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429358) roll requested 2024-11-13T18:27:09,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429370 2024-11-13T18:27:09,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429370 2024-11-13T18:27:09,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429370) roll requested 2024-11-13T18:27:09,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429391 2024-11-13T18:27:09,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429391 2024-11-13T18:27:09,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429391) roll requested 2024-11-13T18:27:09,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429409 2024-11-13T18:27:09,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429409 2024-11-13T18:27:09,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429409) roll requested 2024-11-13T18:27:09,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429425 2024-11-13T18:27:09,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429425 2024-11-13T18:27:09,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429425) roll requested 2024-11-13T18:27:09,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429437 2024-11-13T18:27:09,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429437 2024-11-13T18:27:09,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429437) roll requested 2024-11-13T18:27:09,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429461 2024-11-13T18:27:09,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429461 2024-11-13T18:27:09,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429461) roll requested 2024-11-13T18:27:09,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429483 2024-11-13T18:27:09,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429483 2024-11-13T18:27:09,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429483) roll requested 2024-11-13T18:27:09,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429497 2024-11-13T18:27:09,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429497 2024-11-13T18:27:09,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429497) roll requested 2024-11-13T18:27:09,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429509 2024-11-13T18:27:09,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429509 2024-11-13T18:27:09,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429509) roll requested 2024-11-13T18:27:09,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429518 2024-11-13T18:27:09,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429518 2024-11-13T18:27:09,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429518) roll requested 2024-11-13T18:27:09,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429527 2024-11-13T18:27:09,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429527 2024-11-13T18:27:09,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429527) roll requested 2024-11-13T18:27:09,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429539 2024-11-13T18:27:09,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429539 2024-11-13T18:27:09,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429539) roll requested 2024-11-13T18:27:09,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429548 2024-11-13T18:27:09,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429548 2024-11-13T18:27:09,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429548) roll requested 2024-11-13T18:27:09,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429557 2024-11-13T18:27:09,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429557 2024-11-13T18:27:09,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429557) roll requested 2024-11-13T18:27:09,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429570 2024-11-13T18:27:09,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429570 2024-11-13T18:27:09,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429570) roll requested 2024-11-13T18:27:09,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429578 2024-11-13T18:27:09,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429578 2024-11-13T18:27:09,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429578) roll requested 2024-11-13T18:27:09,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429595 2024-11-13T18:27:09,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429595 2024-11-13T18:27:09,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429595) roll requested 2024-11-13T18:27:09,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429613 2024-11-13T18:27:09,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429613 2024-11-13T18:27:09,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429613) roll requested 2024-11-13T18:27:09,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429623 2024-11-13T18:27:09,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429623 2024-11-13T18:27:09,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429623) roll requested 2024-11-13T18:27:09,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429637 2024-11-13T18:27:09,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429637 2024-11-13T18:27:09,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429637) roll requested 2024-11-13T18:27:09,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429651 2024-11-13T18:27:09,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429651 2024-11-13T18:27:09,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429651) roll requested 2024-11-13T18:27:09,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429668 2024-11-13T18:27:09,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429668 2024-11-13T18:27:09,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429668) roll requested 2024-11-13T18:27:09,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429681 2024-11-13T18:27:09,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429681 2024-11-13T18:27:09,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429681) roll requested 2024-11-13T18:27:09,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429697 2024-11-13T18:27:09,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429697 2024-11-13T18:27:09,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429697) roll requested 2024-11-13T18:27:09,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429710 2024-11-13T18:27:09,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429710 2024-11-13T18:27:09,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429710) roll requested 2024-11-13T18:27:09,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429733 2024-11-13T18:27:09,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429733 2024-11-13T18:27:09,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429733) roll requested 2024-11-13T18:27:09,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429769 2024-11-13T18:27:09,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429769 2024-11-13T18:27:09,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429769) roll requested 2024-11-13T18:27:09,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429789 2024-11-13T18:27:09,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429789 2024-11-13T18:27:09,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429789) roll requested 2024-11-13T18:27:09,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429815 2024-11-13T18:27:09,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429815 2024-11-13T18:27:09,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429815) roll requested 2024-11-13T18:27:09,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429860 2024-11-13T18:27:09,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429860 2024-11-13T18:27:09,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429860) roll requested 2024-11-13T18:27:09,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429909 2024-11-13T18:27:09,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429909 2024-11-13T18:27:09,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429909) roll requested 2024-11-13T18:27:09,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429925 2024-11-13T18:27:09,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429925 2024-11-13T18:27:09,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429925) roll requested 2024-11-13T18:27:09,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429940 2024-11-13T18:27:09,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429940 2024-11-13T18:27:09,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429940) roll requested 2024-11-13T18:27:09,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429949 2024-11-13T18:27:09,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429949 2024-11-13T18:27:09,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429949) roll requested 2024-11-13T18:27:09,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429961 2024-11-13T18:27:09,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429961 2024-11-13T18:27:09,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429961) roll requested 2024-11-13T18:27:09,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429972 2024-11-13T18:27:09,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429972 2024-11-13T18:27:09,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429972) roll requested 2024-11-13T18:27:09,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429982 2024-11-13T18:27:09,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429982 2024-11-13T18:27:09,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:09,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:09,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429982) roll requested 2024-11-13T18:27:09,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522429991 2024-11-13T18:27:09,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:09,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429991 2024-11-13T18:27:10,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522429991) roll requested 2024-11-13T18:27:10,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430001 2024-11-13T18:27:10,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522429991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430001 2024-11-13T18:27:10,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430001) roll requested 2024-11-13T18:27:10,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430008 2024-11-13T18:27:10,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430008 2024-11-13T18:27:10,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430008) roll requested 2024-11-13T18:27:10,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430016 2024-11-13T18:27:10,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430016 2024-11-13T18:27:10,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430016) roll requested 2024-11-13T18:27:10,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430024 2024-11-13T18:27:10,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430024 2024-11-13T18:27:10,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430024) roll requested 2024-11-13T18:27:10,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430037 2024-11-13T18:27:10,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430037 2024-11-13T18:27:10,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430037) roll requested 2024-11-13T18:27:10,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430044 2024-11-13T18:27:10,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430044 2024-11-13T18:27:10,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430044) roll requested 2024-11-13T18:27:10,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430052 2024-11-13T18:27:10,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430052 2024-11-13T18:27:10,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430052) roll requested 2024-11-13T18:27:10,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430065 2024-11-13T18:27:10,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430065 2024-11-13T18:27:10,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430065) roll requested 2024-11-13T18:27:10,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430077 2024-11-13T18:27:10,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430077 2024-11-13T18:27:10,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430077) roll requested 2024-11-13T18:27:10,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430087 2024-11-13T18:27:10,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430087 2024-11-13T18:27:10,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430087) roll requested 2024-11-13T18:27:10,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430101 2024-11-13T18:27:10,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430101 2024-11-13T18:27:10,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430101) roll requested 2024-11-13T18:27:10,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430114 2024-11-13T18:27:10,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430114 2024-11-13T18:27:10,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430114) roll requested 2024-11-13T18:27:10,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430122 2024-11-13T18:27:10,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430122 2024-11-13T18:27:10,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430122) roll requested 2024-11-13T18:27:10,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430130 2024-11-13T18:27:10,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430130 2024-11-13T18:27:10,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430130) roll requested 2024-11-13T18:27:10,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430145 2024-11-13T18:27:10,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430145 2024-11-13T18:27:10,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430145) roll requested 2024-11-13T18:27:10,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430157 2024-11-13T18:27:10,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430157 2024-11-13T18:27:10,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430157) roll requested 2024-11-13T18:27:10,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430179 2024-11-13T18:27:10,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430179 2024-11-13T18:27:10,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430179) roll requested 2024-11-13T18:27:10,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430187 2024-11-13T18:27:10,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430187 2024-11-13T18:27:10,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430187) roll requested 2024-11-13T18:27:10,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430195 2024-11-13T18:27:10,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430195 2024-11-13T18:27:10,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430195) roll requested 2024-11-13T18:27:10,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430202 2024-11-13T18:27:10,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430202 2024-11-13T18:27:10,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430202) roll requested 2024-11-13T18:27:10,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430211 2024-11-13T18:27:10,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430211 2024-11-13T18:27:10,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430211) roll requested 2024-11-13T18:27:10,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430219 2024-11-13T18:27:10,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430219 2024-11-13T18:27:10,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430219) roll requested 2024-11-13T18:27:10,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430226 2024-11-13T18:27:10,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430226 2024-11-13T18:27:10,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430226) roll requested 2024-11-13T18:27:10,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430237 2024-11-13T18:27:10,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430237 2024-11-13T18:27:10,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430237) roll requested 2024-11-13T18:27:10,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430244 2024-11-13T18:27:10,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430244 2024-11-13T18:27:10,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430244) roll requested 2024-11-13T18:27:10,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430251 2024-11-13T18:27:10,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430251 2024-11-13T18:27:10,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430251) roll requested 2024-11-13T18:27:10,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430258 2024-11-13T18:27:10,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430258 2024-11-13T18:27:10,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430258) roll requested 2024-11-13T18:27:10,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430266 2024-11-13T18:27:10,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430266 2024-11-13T18:27:10,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430266) roll requested 2024-11-13T18:27:10,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430275 2024-11-13T18:27:10,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430275 2024-11-13T18:27:10,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430275) roll requested 2024-11-13T18:27:10,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430285 2024-11-13T18:27:10,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430285 2024-11-13T18:27:10,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430285) roll requested 2024-11-13T18:27:10,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430292 2024-11-13T18:27:10,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430292 2024-11-13T18:27:10,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430292) roll requested 2024-11-13T18:27:10,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430305 2024-11-13T18:27:10,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430305 2024-11-13T18:27:10,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430305) roll requested 2024-11-13T18:27:10,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430317 2024-11-13T18:27:10,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430317 2024-11-13T18:27:10,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430317) roll requested 2024-11-13T18:27:10,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430326 2024-11-13T18:27:10,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430326 2024-11-13T18:27:10,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430326) roll requested 2024-11-13T18:27:10,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430337 2024-11-13T18:27:10,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430337 2024-11-13T18:27:10,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430337) roll requested 2024-11-13T18:27:10,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430359 2024-11-13T18:27:10,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430359 2024-11-13T18:27:10,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430359) roll requested 2024-11-13T18:27:10,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430377 2024-11-13T18:27:10,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430377 2024-11-13T18:27:10,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430377) roll requested 2024-11-13T18:27:10,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430393 2024-11-13T18:27:10,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430393 2024-11-13T18:27:10,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430393) roll requested 2024-11-13T18:27:10,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430412 2024-11-13T18:27:10,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430412 2024-11-13T18:27:10,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430412) roll requested 2024-11-13T18:27:10,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430429 2024-11-13T18:27:10,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430429 2024-11-13T18:27:10,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430429) roll requested 2024-11-13T18:27:10,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430441 2024-11-13T18:27:10,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430441 2024-11-13T18:27:10,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430441) roll requested 2024-11-13T18:27:10,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430453 2024-11-13T18:27:10,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430453 2024-11-13T18:27:10,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430453) roll requested 2024-11-13T18:27:10,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430460 2024-11-13T18:27:10,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430460 2024-11-13T18:27:10,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430460) roll requested 2024-11-13T18:27:10,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430470 2024-11-13T18:27:10,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430470 2024-11-13T18:27:10,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430470) roll requested 2024-11-13T18:27:10,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430482 2024-11-13T18:27:10,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430482 2024-11-13T18:27:10,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430482) roll requested 2024-11-13T18:27:10,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430492 2024-11-13T18:27:10,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430492 2024-11-13T18:27:10,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430492) roll requested 2024-11-13T18:27:10,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430501 2024-11-13T18:27:10,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430501 2024-11-13T18:27:10,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430501) roll requested 2024-11-13T18:27:10,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430511 2024-11-13T18:27:10,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430511 2024-11-13T18:27:10,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430511) roll requested 2024-11-13T18:27:10,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430519 2024-11-13T18:27:10,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430519 2024-11-13T18:27:10,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430519) roll requested 2024-11-13T18:27:10,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430527 2024-11-13T18:27:10,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430527 2024-11-13T18:27:10,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430527) roll requested 2024-11-13T18:27:10,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430536 2024-11-13T18:27:10,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430536 2024-11-13T18:27:10,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430536) roll requested 2024-11-13T18:27:10,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430552 2024-11-13T18:27:10,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430552 2024-11-13T18:27:10,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430552) roll requested 2024-11-13T18:27:10,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430563 2024-11-13T18:27:10,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430563 2024-11-13T18:27:10,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430563) roll requested 2024-11-13T18:27:10,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430572 2024-11-13T18:27:10,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430572 2024-11-13T18:27:10,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430572) roll requested 2024-11-13T18:27:10,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430589 2024-11-13T18:27:10,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430589 2024-11-13T18:27:10,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430589) roll requested 2024-11-13T18:27:10,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430598 2024-11-13T18:27:10,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430598 2024-11-13T18:27:10,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430598) roll requested 2024-11-13T18:27:10,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430620 2024-11-13T18:27:10,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430620 2024-11-13T18:27:10,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430620) roll requested 2024-11-13T18:27:10,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430629 2024-11-13T18:27:10,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430629 2024-11-13T18:27:10,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430629) roll requested 2024-11-13T18:27:10,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430645 2024-11-13T18:27:10,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430645 2024-11-13T18:27:10,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430645) roll requested 2024-11-13T18:27:10,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430676 2024-11-13T18:27:10,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430676 2024-11-13T18:27:10,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430676) roll requested 2024-11-13T18:27:10,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430701 2024-11-13T18:27:10,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430701 2024-11-13T18:27:10,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430701) roll requested 2024-11-13T18:27:10,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430729 2024-11-13T18:27:10,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430729 2024-11-13T18:27:10,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430729) roll requested 2024-11-13T18:27:10,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430747 2024-11-13T18:27:10,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430747 2024-11-13T18:27:10,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430747) roll requested 2024-11-13T18:27:10,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430761 2024-11-13T18:27:10,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430761 2024-11-13T18:27:10,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430761) roll requested 2024-11-13T18:27:10,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430781 2024-11-13T18:27:10,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430781 2024-11-13T18:27:10,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430781) roll requested 2024-11-13T18:27:10,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430790 2024-11-13T18:27:10,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430790 2024-11-13T18:27:10,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430790) roll requested 2024-11-13T18:27:10,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430800 2024-11-13T18:27:10,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430800 2024-11-13T18:27:10,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430800) roll requested 2024-11-13T18:27:10,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430808 2024-11-13T18:27:10,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430808 2024-11-13T18:27:10,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430808) roll requested 2024-11-13T18:27:10,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430817 2024-11-13T18:27:10,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430817 2024-11-13T18:27:10,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430817) roll requested 2024-11-13T18:27:10,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430829 2024-11-13T18:27:10,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430829 2024-11-13T18:27:10,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430829) roll requested 2024-11-13T18:27:10,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430845 2024-11-13T18:27:10,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430845 2024-11-13T18:27:10,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430845) roll requested 2024-11-13T18:27:10,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430857 2024-11-13T18:27:10,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430857 2024-11-13T18:27:10,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430857) roll requested 2024-11-13T18:27:10,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430865 2024-11-13T18:27:10,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430865 2024-11-13T18:27:10,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430865) roll requested 2024-11-13T18:27:10,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430873 2024-11-13T18:27:10,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430873 2024-11-13T18:27:10,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430873) roll requested 2024-11-13T18:27:10,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430889 2024-11-13T18:27:10,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430889 2024-11-13T18:27:10,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430889) roll requested 2024-11-13T18:27:10,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430907 2024-11-13T18:27:10,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430907 2024-11-13T18:27:10,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430907) roll requested 2024-11-13T18:27:10,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430932 2024-11-13T18:27:10,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430932 2024-11-13T18:27:10,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430932) roll requested 2024-11-13T18:27:10,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430957 2024-11-13T18:27:10,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430957 2024-11-13T18:27:10,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430957) roll requested 2024-11-13T18:27:10,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430974 2024-11-13T18:27:10,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430974 2024-11-13T18:27:10,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:10,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:10,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430974) roll requested 2024-11-13T18:27:10,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522430983 2024-11-13T18:27:10,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:10,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430983 2024-11-13T18:27:11,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522430983) roll requested 2024-11-13T18:27:11,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431000 2024-11-13T18:27:11,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522430983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431000 2024-11-13T18:27:11,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431000) roll requested 2024-11-13T18:27:11,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431013 2024-11-13T18:27:11,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431013 2024-11-13T18:27:11,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431013) roll requested 2024-11-13T18:27:11,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431025 2024-11-13T18:27:11,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431025 2024-11-13T18:27:11,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431025) roll requested 2024-11-13T18:27:11,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431036 2024-11-13T18:27:11,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431036 2024-11-13T18:27:11,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431036) roll requested 2024-11-13T18:27:11,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431050 2024-11-13T18:27:11,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431050 2024-11-13T18:27:11,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431050) roll requested 2024-11-13T18:27:11,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431058 2024-11-13T18:27:11,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431058 2024-11-13T18:27:11,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431058) roll requested 2024-11-13T18:27:11,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431065 2024-11-13T18:27:11,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431065 2024-11-13T18:27:11,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431065) roll requested 2024-11-13T18:27:11,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431071 2024-11-13T18:27:11,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431071 2024-11-13T18:27:11,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431071) roll requested 2024-11-13T18:27:11,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431081 2024-11-13T18:27:11,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431081 2024-11-13T18:27:11,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431081) roll requested 2024-11-13T18:27:11,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431090 2024-11-13T18:27:11,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431090 2024-11-13T18:27:11,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431090) roll requested 2024-11-13T18:27:11,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431101 2024-11-13T18:27:11,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431101 2024-11-13T18:27:11,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431101) roll requested 2024-11-13T18:27:11,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431117 2024-11-13T18:27:11,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431117 2024-11-13T18:27:11,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431117) roll requested 2024-11-13T18:27:11,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431128 2024-11-13T18:27:11,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431128 2024-11-13T18:27:11,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431128) roll requested 2024-11-13T18:27:11,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431137 2024-11-13T18:27:11,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431137 2024-11-13T18:27:11,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431137) roll requested 2024-11-13T18:27:11,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431149 2024-11-13T18:27:11,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431149 2024-11-13T18:27:11,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431149) roll requested 2024-11-13T18:27:11,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431158 2024-11-13T18:27:11,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431158 2024-11-13T18:27:11,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431158) roll requested 2024-11-13T18:27:11,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431169 2024-11-13T18:27:11,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431169 2024-11-13T18:27:11,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431169) roll requested 2024-11-13T18:27:11,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431178 2024-11-13T18:27:11,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431178 2024-11-13T18:27:11,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431178) roll requested 2024-11-13T18:27:11,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431187 2024-11-13T18:27:11,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431187 2024-11-13T18:27:11,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431187) roll requested 2024-11-13T18:27:11,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431196 2024-11-13T18:27:11,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431196 2024-11-13T18:27:11,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431196) roll requested 2024-11-13T18:27:11,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431204 2024-11-13T18:27:11,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431204 2024-11-13T18:27:11,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431204) roll requested 2024-11-13T18:27:11,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431212 2024-11-13T18:27:11,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431212 2024-11-13T18:27:11,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431212) roll requested 2024-11-13T18:27:11,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431225 2024-11-13T18:27:11,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431225 2024-11-13T18:27:11,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431225) roll requested 2024-11-13T18:27:11,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431248 2024-11-13T18:27:11,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431248 2024-11-13T18:27:11,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431248) roll requested 2024-11-13T18:27:11,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431261 2024-11-13T18:27:11,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431261 2024-11-13T18:27:11,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431261) roll requested 2024-11-13T18:27:11,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431269 2024-11-13T18:27:11,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431269 2024-11-13T18:27:11,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431269) roll requested 2024-11-13T18:27:11,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431278 2024-11-13T18:27:11,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431278 2024-11-13T18:27:11,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431278) roll requested 2024-11-13T18:27:11,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431285 2024-11-13T18:27:11,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431285 2024-11-13T18:27:11,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431285) roll requested 2024-11-13T18:27:11,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431292 2024-11-13T18:27:11,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431292 2024-11-13T18:27:11,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431292) roll requested 2024-11-13T18:27:11,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431300 2024-11-13T18:27:11,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431300 2024-11-13T18:27:11,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431300) roll requested 2024-11-13T18:27:11,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431307 2024-11-13T18:27:11,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431307 2024-11-13T18:27:11,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431307) roll requested 2024-11-13T18:27:11,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431315 2024-11-13T18:27:11,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431315 2024-11-13T18:27:11,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431315) roll requested 2024-11-13T18:27:11,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431322 2024-11-13T18:27:11,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431322 2024-11-13T18:27:11,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431322) roll requested 2024-11-13T18:27:11,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431329 2024-11-13T18:27:11,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431329 2024-11-13T18:27:11,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431329) roll requested 2024-11-13T18:27:11,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431337 2024-11-13T18:27:11,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431337 2024-11-13T18:27:11,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431337) roll requested 2024-11-13T18:27:11,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431346 2024-11-13T18:27:11,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431346 2024-11-13T18:27:11,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431346) roll requested 2024-11-13T18:27:11,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431357 2024-11-13T18:27:11,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431357 2024-11-13T18:27:11,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431357) roll requested 2024-11-13T18:27:11,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431364 2024-11-13T18:27:11,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431364 2024-11-13T18:27:11,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431364) roll requested 2024-11-13T18:27:11,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431372 2024-11-13T18:27:11,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431372 2024-11-13T18:27:11,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431372) roll requested 2024-11-13T18:27:11,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431379 2024-11-13T18:27:11,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431379 2024-11-13T18:27:11,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431379) roll requested 2024-11-13T18:27:11,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431387 2024-11-13T18:27:11,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431387 2024-11-13T18:27:11,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431387) roll requested 2024-11-13T18:27:11,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431395 2024-11-13T18:27:11,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431395 2024-11-13T18:27:11,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431395) roll requested 2024-11-13T18:27:11,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431404 2024-11-13T18:27:11,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431404 2024-11-13T18:27:11,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431404) roll requested 2024-11-13T18:27:11,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431411 2024-11-13T18:27:11,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431411 2024-11-13T18:27:11,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431411) roll requested 2024-11-13T18:27:11,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431421 2024-11-13T18:27:11,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431421 2024-11-13T18:27:11,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431421) roll requested 2024-11-13T18:27:11,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431428 2024-11-13T18:27:11,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431428 2024-11-13T18:27:11,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431428) roll requested 2024-11-13T18:27:11,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431435 2024-11-13T18:27:11,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431435 2024-11-13T18:27:11,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431435) roll requested 2024-11-13T18:27:11,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431446 2024-11-13T18:27:11,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431446 2024-11-13T18:27:11,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431446) roll requested 2024-11-13T18:27:11,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431457 2024-11-13T18:27:11,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431457 2024-11-13T18:27:11,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431457) roll requested 2024-11-13T18:27:11,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431473 2024-11-13T18:27:11,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431473 2024-11-13T18:27:11,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431473) roll requested 2024-11-13T18:27:11,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431484 2024-11-13T18:27:11,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431484 2024-11-13T18:27:11,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431484) roll requested 2024-11-13T18:27:11,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431497 2024-11-13T18:27:11,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431497 2024-11-13T18:27:11,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431497) roll requested 2024-11-13T18:27:11,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431517 2024-11-13T18:27:11,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431517 2024-11-13T18:27:11,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431517) roll requested 2024-11-13T18:27:11,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431528 2024-11-13T18:27:11,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431528 2024-11-13T18:27:11,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431528) roll requested 2024-11-13T18:27:11,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431541 2024-11-13T18:27:11,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431541 2024-11-13T18:27:11,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431541) roll requested 2024-11-13T18:27:11,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431552 2024-11-13T18:27:11,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431552 2024-11-13T18:27:11,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431552) roll requested 2024-11-13T18:27:11,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431565 2024-11-13T18:27:11,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431565 2024-11-13T18:27:11,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431565) roll requested 2024-11-13T18:27:11,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431577 2024-11-13T18:27:11,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431577 2024-11-13T18:27:11,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431577) roll requested 2024-11-13T18:27:11,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431585 2024-11-13T18:27:11,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431585 2024-11-13T18:27:11,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431585) roll requested 2024-11-13T18:27:11,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431592 2024-11-13T18:27:11,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431592 2024-11-13T18:27:11,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431592) roll requested 2024-11-13T18:27:11,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431603 2024-11-13T18:27:11,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431603 2024-11-13T18:27:11,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431603) roll requested 2024-11-13T18:27:11,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431613 2024-11-13T18:27:11,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431613 2024-11-13T18:27:11,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431613) roll requested 2024-11-13T18:27:11,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431622 2024-11-13T18:27:11,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431622 2024-11-13T18:27:11,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431622) roll requested 2024-11-13T18:27:11,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431629 2024-11-13T18:27:11,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431629 2024-11-13T18:27:11,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431629) roll requested 2024-11-13T18:27:11,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431638 2024-11-13T18:27:11,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431638 2024-11-13T18:27:11,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431638) roll requested 2024-11-13T18:27:11,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431645 2024-11-13T18:27:11,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431645 2024-11-13T18:27:11,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431645) roll requested 2024-11-13T18:27:11,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431657 2024-11-13T18:27:11,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431657 2024-11-13T18:27:11,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431657) roll requested 2024-11-13T18:27:11,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431673 2024-11-13T18:27:11,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431673 2024-11-13T18:27:11,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431673) roll requested 2024-11-13T18:27:11,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431683 2024-11-13T18:27:11,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431683 2024-11-13T18:27:11,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431683) roll requested 2024-11-13T18:27:11,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431697 2024-11-13T18:27:11,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431697 2024-11-13T18:27:11,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431697) roll requested 2024-11-13T18:27:11,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431705 2024-11-13T18:27:11,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431705 2024-11-13T18:27:11,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431705) roll requested 2024-11-13T18:27:11,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431717 2024-11-13T18:27:11,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431717 2024-11-13T18:27:11,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431717) roll requested 2024-11-13T18:27:11,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431726 2024-11-13T18:27:11,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431726 2024-11-13T18:27:11,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431726) roll requested 2024-11-13T18:27:11,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431737 2024-11-13T18:27:11,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431737 2024-11-13T18:27:11,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431737) roll requested 2024-11-13T18:27:11,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431745 2024-11-13T18:27:11,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431745 2024-11-13T18:27:11,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431745) roll requested 2024-11-13T18:27:11,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431757 2024-11-13T18:27:11,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431757 2024-11-13T18:27:11,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431757) roll requested 2024-11-13T18:27:11,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431768 2024-11-13T18:27:11,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431768 2024-11-13T18:27:11,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431768) roll requested 2024-11-13T18:27:11,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431776 2024-11-13T18:27:11,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431776 2024-11-13T18:27:11,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431776) roll requested 2024-11-13T18:27:11,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431782 2024-11-13T18:27:11,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431782 2024-11-13T18:27:11,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431782) roll requested 2024-11-13T18:27:11,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431790 2024-11-13T18:27:11,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431790 2024-11-13T18:27:11,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431790) roll requested 2024-11-13T18:27:11,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431798 2024-11-13T18:27:11,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431798 2024-11-13T18:27:11,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431798) roll requested 2024-11-13T18:27:11,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431809 2024-11-13T18:27:11,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431809 2024-11-13T18:27:11,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431809) roll requested 2024-11-13T18:27:11,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431818 2024-11-13T18:27:11,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431818 2024-11-13T18:27:11,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431818) roll requested 2024-11-13T18:27:11,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431832 2024-11-13T18:27:11,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431832 2024-11-13T18:27:11,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431832) roll requested 2024-11-13T18:27:11,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431847 2024-11-13T18:27:11,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431847 2024-11-13T18:27:11,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431847) roll requested 2024-11-13T18:27:11,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431857 2024-11-13T18:27:11,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431857 2024-11-13T18:27:11,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431857) roll requested 2024-11-13T18:27:11,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431874 2024-11-13T18:27:11,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431874 2024-11-13T18:27:11,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431874) roll requested 2024-11-13T18:27:11,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431885 2024-11-13T18:27:11,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431885 2024-11-13T18:27:11,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431885) roll requested 2024-11-13T18:27:11,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431895 2024-11-13T18:27:11,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431895 2024-11-13T18:27:11,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431895) roll requested 2024-11-13T18:27:11,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431903 2024-11-13T18:27:11,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431903 2024-11-13T18:27:11,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431903) roll requested 2024-11-13T18:27:11,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431911 2024-11-13T18:27:11,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431911 2024-11-13T18:27:11,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431911) roll requested 2024-11-13T18:27:11,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431920 2024-11-13T18:27:11,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431920 2024-11-13T18:27:11,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431920) roll requested 2024-11-13T18:27:11,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431927 2024-11-13T18:27:11,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431927 2024-11-13T18:27:11,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431927) roll requested 2024-11-13T18:27:11,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431935 2024-11-13T18:27:11,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431935 2024-11-13T18:27:11,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431935) roll requested 2024-11-13T18:27:11,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431945 2024-11-13T18:27:11,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431945 2024-11-13T18:27:11,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431945) roll requested 2024-11-13T18:27:11,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431956 2024-11-13T18:27:11,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431956 2024-11-13T18:27:11,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431956) roll requested 2024-11-13T18:27:11,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431968 2024-11-13T18:27:11,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431968 2024-11-13T18:27:11,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431968) roll requested 2024-11-13T18:27:11,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431979 2024-11-13T18:27:11,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431979 2024-11-13T18:27:11,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431979) roll requested 2024-11-13T18:27:11,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431986 2024-11-13T18:27:11,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431986 2024-11-13T18:27:11,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431986) roll requested 2024-11-13T18:27:11,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522431993 2024-11-13T18:27:11,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:11,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431993 2024-11-13T18:27:11,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:11,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:11,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522431993) roll requested 2024-11-13T18:27:12,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432000 2024-11-13T18:27:12,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522431993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432000 2024-11-13T18:27:12,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432000) roll requested 2024-11-13T18:27:12,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432007 2024-11-13T18:27:12,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432007 2024-11-13T18:27:12,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432007) roll requested 2024-11-13T18:27:12,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432015 2024-11-13T18:27:12,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432015 2024-11-13T18:27:12,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432015) roll requested 2024-11-13T18:27:12,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432022 2024-11-13T18:27:12,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432022 2024-11-13T18:27:12,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432022) roll requested 2024-11-13T18:27:12,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432029 2024-11-13T18:27:12,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432029 2024-11-13T18:27:12,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432029) roll requested 2024-11-13T18:27:12,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432040 2024-11-13T18:27:12,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432040 2024-11-13T18:27:12,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432040) roll requested 2024-11-13T18:27:12,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432051 2024-11-13T18:27:12,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432051 2024-11-13T18:27:12,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432051) roll requested 2024-11-13T18:27:12,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432061 2024-11-13T18:27:12,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432061 2024-11-13T18:27:12,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432061) roll requested 2024-11-13T18:27:12,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432075 2024-11-13T18:27:12,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432075 2024-11-13T18:27:12,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432075) roll requested 2024-11-13T18:27:12,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432083 2024-11-13T18:27:12,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432083 2024-11-13T18:27:12,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432083) roll requested 2024-11-13T18:27:12,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432091 2024-11-13T18:27:12,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432091 2024-11-13T18:27:12,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432091) roll requested 2024-11-13T18:27:12,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432099 2024-11-13T18:27:12,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432099 2024-11-13T18:27:12,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432099) roll requested 2024-11-13T18:27:12,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432106 2024-11-13T18:27:12,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432106 2024-11-13T18:27:12,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432106) roll requested 2024-11-13T18:27:12,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432113 2024-11-13T18:27:12,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432113 2024-11-13T18:27:12,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432113) roll requested 2024-11-13T18:27:12,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432121 2024-11-13T18:27:12,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432121 2024-11-13T18:27:12,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432121) roll requested 2024-11-13T18:27:12,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432130 2024-11-13T18:27:12,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432130 2024-11-13T18:27:12,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432130) roll requested 2024-11-13T18:27:12,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432137 2024-11-13T18:27:12,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432137 2024-11-13T18:27:12,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432137) roll requested 2024-11-13T18:27:12,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432144 2024-11-13T18:27:12,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432144 2024-11-13T18:27:12,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432144) roll requested 2024-11-13T18:27:12,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432150 2024-11-13T18:27:12,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432150 2024-11-13T18:27:12,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432150) roll requested 2024-11-13T18:27:12,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432157 2024-11-13T18:27:12,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432157 2024-11-13T18:27:12,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432157) roll requested 2024-11-13T18:27:12,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432167 2024-11-13T18:27:12,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432167 2024-11-13T18:27:12,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432167) roll requested 2024-11-13T18:27:12,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432174 2024-11-13T18:27:12,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432174 2024-11-13T18:27:12,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432174) roll requested 2024-11-13T18:27:12,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432182 2024-11-13T18:27:12,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432182 2024-11-13T18:27:12,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432182) roll requested 2024-11-13T18:27:12,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432190 2024-11-13T18:27:12,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432190 2024-11-13T18:27:12,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432190) roll requested 2024-11-13T18:27:12,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432197 2024-11-13T18:27:12,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432197 2024-11-13T18:27:12,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432197) roll requested 2024-11-13T18:27:12,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432207 2024-11-13T18:27:12,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432207 2024-11-13T18:27:12,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432207) roll requested 2024-11-13T18:27:12,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432215 2024-11-13T18:27:12,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432215 2024-11-13T18:27:12,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432215) roll requested 2024-11-13T18:27:12,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432223 2024-11-13T18:27:12,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432223 2024-11-13T18:27:12,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432223) roll requested 2024-11-13T18:27:12,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432229 2024-11-13T18:27:12,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432229 2024-11-13T18:27:12,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432229) roll requested 2024-11-13T18:27:12,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432251 2024-11-13T18:27:12,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432251 2024-11-13T18:27:12,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432251) roll requested 2024-11-13T18:27:12,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432259 2024-11-13T18:27:12,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432259 2024-11-13T18:27:12,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432259) roll requested 2024-11-13T18:27:12,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432266 2024-11-13T18:27:12,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432266 2024-11-13T18:27:12,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432266) roll requested 2024-11-13T18:27:12,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432273 2024-11-13T18:27:12,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432273 2024-11-13T18:27:12,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432273) roll requested 2024-11-13T18:27:12,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432281 2024-11-13T18:27:12,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432281 2024-11-13T18:27:12,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432281) roll requested 2024-11-13T18:27:12,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432290 2024-11-13T18:27:12,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432290 2024-11-13T18:27:12,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432290) roll requested 2024-11-13T18:27:12,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432299 2024-11-13T18:27:12,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432299 2024-11-13T18:27:12,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432299) roll requested 2024-11-13T18:27:12,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432308 2024-11-13T18:27:12,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432308 2024-11-13T18:27:12,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432308) roll requested 2024-11-13T18:27:12,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432316 2024-11-13T18:27:12,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432316 2024-11-13T18:27:12,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432316) roll requested 2024-11-13T18:27:12,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432325 2024-11-13T18:27:12,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432325 2024-11-13T18:27:12,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432325) roll requested 2024-11-13T18:27:12,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432333 2024-11-13T18:27:12,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432333 2024-11-13T18:27:12,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432333) roll requested 2024-11-13T18:27:12,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432344 2024-11-13T18:27:12,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432344 2024-11-13T18:27:12,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432344) roll requested 2024-11-13T18:27:12,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432352 2024-11-13T18:27:12,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432352 2024-11-13T18:27:12,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432352) roll requested 2024-11-13T18:27:12,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432360 2024-11-13T18:27:12,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432360 2024-11-13T18:27:12,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432360) roll requested 2024-11-13T18:27:12,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432368 2024-11-13T18:27:12,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432368 2024-11-13T18:27:12,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432368) roll requested 2024-11-13T18:27:12,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432375 2024-11-13T18:27:12,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432375 2024-11-13T18:27:12,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432375) roll requested 2024-11-13T18:27:12,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432383 2024-11-13T18:27:12,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432383 2024-11-13T18:27:12,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432383) roll requested 2024-11-13T18:27:12,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432391 2024-11-13T18:27:12,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432391 2024-11-13T18:27:12,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432391) roll requested 2024-11-13T18:27:12,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432399 2024-11-13T18:27:12,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432399 2024-11-13T18:27:12,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432399) roll requested 2024-11-13T18:27:12,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432406 2024-11-13T18:27:12,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432406 2024-11-13T18:27:12,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432406) roll requested 2024-11-13T18:27:12,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432414 2024-11-13T18:27:12,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432414 2024-11-13T18:27:12,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432414) roll requested 2024-11-13T18:27:12,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432422 2024-11-13T18:27:12,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432422 2024-11-13T18:27:12,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432422) roll requested 2024-11-13T18:27:12,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432429 2024-11-13T18:27:12,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432429 2024-11-13T18:27:12,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432429) roll requested 2024-11-13T18:27:12,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432440 2024-11-13T18:27:12,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432440 2024-11-13T18:27:12,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432440) roll requested 2024-11-13T18:27:12,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432453 2024-11-13T18:27:12,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432453 2024-11-13T18:27:12,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432453) roll requested 2024-11-13T18:27:12,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432465 2024-11-13T18:27:12,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432465 2024-11-13T18:27:12,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432465) roll requested 2024-11-13T18:27:12,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432473 2024-11-13T18:27:12,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432473 2024-11-13T18:27:12,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432473) roll requested 2024-11-13T18:27:12,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432493 2024-11-13T18:27:12,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432493 2024-11-13T18:27:12,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432493) roll requested 2024-11-13T18:27:12,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432502 2024-11-13T18:27:12,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432502 2024-11-13T18:27:12,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432502) roll requested 2024-11-13T18:27:12,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432514 2024-11-13T18:27:12,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432514 2024-11-13T18:27:12,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432514) roll requested 2024-11-13T18:27:12,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432528 2024-11-13T18:27:12,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432528 2024-11-13T18:27:12,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432528) roll requested 2024-11-13T18:27:12,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432537 2024-11-13T18:27:12,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432537 2024-11-13T18:27:12,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432537) roll requested 2024-11-13T18:27:12,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432545 2024-11-13T18:27:12,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432545 2024-11-13T18:27:12,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432545) roll requested 2024-11-13T18:27:12,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432553 2024-11-13T18:27:12,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432553 2024-11-13T18:27:12,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432553) roll requested 2024-11-13T18:27:12,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432569 2024-11-13T18:27:12,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432569 2024-11-13T18:27:12,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432569) roll requested 2024-11-13T18:27:12,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432584 2024-11-13T18:27:12,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432584 2024-11-13T18:27:12,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432584) roll requested 2024-11-13T18:27:12,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432597 2024-11-13T18:27:12,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432597 2024-11-13T18:27:12,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432597) roll requested 2024-11-13T18:27:12,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432607 2024-11-13T18:27:12,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432607 2024-11-13T18:27:12,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432607) roll requested 2024-11-13T18:27:12,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432616 2024-11-13T18:27:12,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432616 2024-11-13T18:27:12,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432616) roll requested 2024-11-13T18:27:12,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432627 2024-11-13T18:27:12,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432627 2024-11-13T18:27:12,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432627) roll requested 2024-11-13T18:27:12,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432636 2024-11-13T18:27:12,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432636 2024-11-13T18:27:12,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432636) roll requested 2024-11-13T18:27:12,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432646 2024-11-13T18:27:12,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432646 2024-11-13T18:27:12,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432646) roll requested 2024-11-13T18:27:12,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432654 2024-11-13T18:27:12,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432654 2024-11-13T18:27:12,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432654) roll requested 2024-11-13T18:27:12,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432661 2024-11-13T18:27:12,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432661 2024-11-13T18:27:12,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432661) roll requested 2024-11-13T18:27:12,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432669 2024-11-13T18:27:12,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432669 2024-11-13T18:27:12,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432669) roll requested 2024-11-13T18:27:12,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432682 2024-11-13T18:27:12,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432682 2024-11-13T18:27:12,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432682) roll requested 2024-11-13T18:27:12,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432691 2024-11-13T18:27:12,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432691 2024-11-13T18:27:12,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432691) roll requested 2024-11-13T18:27:12,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432700 2024-11-13T18:27:12,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432700 2024-11-13T18:27:12,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432700) roll requested 2024-11-13T18:27:12,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432708 2024-11-13T18:27:12,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432708 2024-11-13T18:27:12,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432708) roll requested 2024-11-13T18:27:12,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432717 2024-11-13T18:27:12,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432717 2024-11-13T18:27:12,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432717) roll requested 2024-11-13T18:27:12,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432725 2024-11-13T18:27:12,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432725 2024-11-13T18:27:12,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432725) roll requested 2024-11-13T18:27:12,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432732 2024-11-13T18:27:12,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432732 2024-11-13T18:27:12,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432732) roll requested 2024-11-13T18:27:12,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432741 2024-11-13T18:27:12,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432741 2024-11-13T18:27:12,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432741) roll requested 2024-11-13T18:27:12,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432749 2024-11-13T18:27:12,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432749 2024-11-13T18:27:12,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432749) roll requested 2024-11-13T18:27:12,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432761 2024-11-13T18:27:12,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432761 2024-11-13T18:27:12,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432761) roll requested 2024-11-13T18:27:12,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432773 2024-11-13T18:27:12,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432773 2024-11-13T18:27:12,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432773) roll requested 2024-11-13T18:27:12,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432793 2024-11-13T18:27:12,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432793 2024-11-13T18:27:12,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432793) roll requested 2024-11-13T18:27:12,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432816 2024-11-13T18:27:12,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432816 2024-11-13T18:27:12,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432816) roll requested 2024-11-13T18:27:12,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432832 2024-11-13T18:27:12,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432832 2024-11-13T18:27:12,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432832) roll requested 2024-11-13T18:27:12,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432849 2024-11-13T18:27:12,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432849 2024-11-13T18:27:12,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432849) roll requested 2024-11-13T18:27:12,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432873 2024-11-13T18:27:12,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432873 2024-11-13T18:27:12,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432873) roll requested 2024-11-13T18:27:12,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432901 2024-11-13T18:27:12,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432901 2024-11-13T18:27:12,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432901) roll requested 2024-11-13T18:27:12,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432949 2024-11-13T18:27:12,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432949 2024-11-13T18:27:12,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432949) roll requested 2024-11-13T18:27:12,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432981 2024-11-13T18:27:12,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:12,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432981 2024-11-13T18:27:12,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:12,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:12,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432981) roll requested 2024-11-13T18:27:12,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522432995 2024-11-13T18:27:13,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432995 2024-11-13T18:27:13,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522432995) roll requested 2024-11-13T18:27:13,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433009 2024-11-13T18:27:13,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522432995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433009 2024-11-13T18:27:13,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433009) roll requested 2024-11-13T18:27:13,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433038 2024-11-13T18:27:13,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433038 2024-11-13T18:27:13,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433038) roll requested 2024-11-13T18:27:13,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433055 2024-11-13T18:27:13,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433055 2024-11-13T18:27:13,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433055) roll requested 2024-11-13T18:27:13,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433069 2024-11-13T18:27:13,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433069 2024-11-13T18:27:13,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433069) roll requested 2024-11-13T18:27:13,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433079 2024-11-13T18:27:13,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433079 2024-11-13T18:27:13,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433079) roll requested 2024-11-13T18:27:13,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433089 2024-11-13T18:27:13,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433089 2024-11-13T18:27:13,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433089) roll requested 2024-11-13T18:27:13,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433099 2024-11-13T18:27:13,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433099 2024-11-13T18:27:13,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433099) roll requested 2024-11-13T18:27:13,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433108 2024-11-13T18:27:13,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433108 2024-11-13T18:27:13,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433108) roll requested 2024-11-13T18:27:13,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433125 2024-11-13T18:27:13,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433125 2024-11-13T18:27:13,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433125) roll requested 2024-11-13T18:27:13,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433137 2024-11-13T18:27:13,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433137 2024-11-13T18:27:13,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433137) roll requested 2024-11-13T18:27:13,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433151 2024-11-13T18:27:13,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433151 2024-11-13T18:27:13,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433151) roll requested 2024-11-13T18:27:13,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433167 2024-11-13T18:27:13,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433167 2024-11-13T18:27:13,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433167) roll requested 2024-11-13T18:27:13,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433177 2024-11-13T18:27:13,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433177 2024-11-13T18:27:13,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433177) roll requested 2024-11-13T18:27:13,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433183 2024-11-13T18:27:13,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433183 2024-11-13T18:27:13,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433183) roll requested 2024-11-13T18:27:13,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433190 2024-11-13T18:27:13,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433190 2024-11-13T18:27:13,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433190) roll requested 2024-11-13T18:27:13,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433197 2024-11-13T18:27:13,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433197 2024-11-13T18:27:13,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433197) roll requested 2024-11-13T18:27:13,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433203 2024-11-13T18:27:13,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433203 2024-11-13T18:27:13,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433203) roll requested 2024-11-13T18:27:13,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433210 2024-11-13T18:27:13,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433210 2024-11-13T18:27:13,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433210) roll requested 2024-11-13T18:27:13,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433223 2024-11-13T18:27:13,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433223 2024-11-13T18:27:13,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433223) roll requested 2024-11-13T18:27:13,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433231 2024-11-13T18:27:13,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433231 2024-11-13T18:27:13,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433231) roll requested 2024-11-13T18:27:13,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433240 2024-11-13T18:27:13,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433240 2024-11-13T18:27:13,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433240) roll requested 2024-11-13T18:27:13,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433248 2024-11-13T18:27:13,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433248 2024-11-13T18:27:13,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433248) roll requested 2024-11-13T18:27:13,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433254 2024-11-13T18:27:13,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433254 2024-11-13T18:27:13,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433254) roll requested 2024-11-13T18:27:13,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433263 2024-11-13T18:27:13,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433263 2024-11-13T18:27:13,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433263) roll requested 2024-11-13T18:27:13,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433270 2024-11-13T18:27:13,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433270 2024-11-13T18:27:13,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433270) roll requested 2024-11-13T18:27:13,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433276 2024-11-13T18:27:13,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433276 2024-11-13T18:27:13,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433276) roll requested 2024-11-13T18:27:13,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433284 2024-11-13T18:27:13,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433284 2024-11-13T18:27:13,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433284) roll requested 2024-11-13T18:27:13,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433293 2024-11-13T18:27:13,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433293 2024-11-13T18:27:13,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433293) roll requested 2024-11-13T18:27:13,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433305 2024-11-13T18:27:13,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433305 2024-11-13T18:27:13,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433305) roll requested 2024-11-13T18:27:13,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433313 2024-11-13T18:27:13,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433313 2024-11-13T18:27:13,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433313) roll requested 2024-11-13T18:27:13,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433320 2024-11-13T18:27:13,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433320 2024-11-13T18:27:13,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433320) roll requested 2024-11-13T18:27:13,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433328 2024-11-13T18:27:13,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433328 2024-11-13T18:27:13,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433328) roll requested 2024-11-13T18:27:13,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433336 2024-11-13T18:27:13,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433336 2024-11-13T18:27:13,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433336) roll requested 2024-11-13T18:27:13,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433352 2024-11-13T18:27:13,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433352 2024-11-13T18:27:13,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433352) roll requested 2024-11-13T18:27:13,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433361 2024-11-13T18:27:13,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433361 2024-11-13T18:27:13,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433361) roll requested 2024-11-13T18:27:13,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433373 2024-11-13T18:27:13,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433373 2024-11-13T18:27:13,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433373) roll requested 2024-11-13T18:27:13,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433381 2024-11-13T18:27:13,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433381 2024-11-13T18:27:13,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433381) roll requested 2024-11-13T18:27:13,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433392 2024-11-13T18:27:13,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433392 2024-11-13T18:27:13,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433392) roll requested 2024-11-13T18:27:13,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433400 2024-11-13T18:27:13,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433400 2024-11-13T18:27:13,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433400) roll requested 2024-11-13T18:27:13,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433407 2024-11-13T18:27:13,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433407 2024-11-13T18:27:13,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433407) roll requested 2024-11-13T18:27:13,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433415 2024-11-13T18:27:13,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433415 2024-11-13T18:27:13,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433415) roll requested 2024-11-13T18:27:13,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433423 2024-11-13T18:27:13,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433423 2024-11-13T18:27:13,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433423) roll requested 2024-11-13T18:27:13,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433430 2024-11-13T18:27:13,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433430 2024-11-13T18:27:13,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433430) roll requested 2024-11-13T18:27:13,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433436 2024-11-13T18:27:13,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433436 2024-11-13T18:27:13,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433436) roll requested 2024-11-13T18:27:13,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433453 2024-11-13T18:27:13,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433453 2024-11-13T18:27:13,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433453) roll requested 2024-11-13T18:27:13,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433460 2024-11-13T18:27:13,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433460 2024-11-13T18:27:13,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433460) roll requested 2024-11-13T18:27:13,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433472 2024-11-13T18:27:13,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433472 2024-11-13T18:27:13,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433472) roll requested 2024-11-13T18:27:13,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433479 2024-11-13T18:27:13,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433479 2024-11-13T18:27:13,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433479) roll requested 2024-11-13T18:27:13,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433488 2024-11-13T18:27:13,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433488 2024-11-13T18:27:13,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433488) roll requested 2024-11-13T18:27:13,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433498 2024-11-13T18:27:13,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433498 2024-11-13T18:27:13,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433498) roll requested 2024-11-13T18:27:13,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433517 2024-11-13T18:27:13,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433517 2024-11-13T18:27:13,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433517) roll requested 2024-11-13T18:27:13,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433524 2024-11-13T18:27:13,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433524 2024-11-13T18:27:13,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433524) roll requested 2024-11-13T18:27:13,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433534 2024-11-13T18:27:13,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433534 2024-11-13T18:27:13,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433534) roll requested 2024-11-13T18:27:13,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433553 2024-11-13T18:27:13,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433553 2024-11-13T18:27:13,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433553) roll requested 2024-11-13T18:27:13,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433560 2024-11-13T18:27:13,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433560 2024-11-13T18:27:13,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433560) roll requested 2024-11-13T18:27:13,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433568 2024-11-13T18:27:13,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433568 2024-11-13T18:27:13,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433568) roll requested 2024-11-13T18:27:13,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433575 2024-11-13T18:27:13,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433575 2024-11-13T18:27:13,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433575) roll requested 2024-11-13T18:27:13,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433587 2024-11-13T18:27:13,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433587 2024-11-13T18:27:13,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433587) roll requested 2024-11-13T18:27:13,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433595 2024-11-13T18:27:13,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433595 2024-11-13T18:27:13,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433595) roll requested 2024-11-13T18:27:13,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433602 2024-11-13T18:27:13,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433602 2024-11-13T18:27:13,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433602) roll requested 2024-11-13T18:27:13,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433609 2024-11-13T18:27:13,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433609 2024-11-13T18:27:13,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433609) roll requested 2024-11-13T18:27:13,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433617 2024-11-13T18:27:13,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433617 2024-11-13T18:27:13,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433617) roll requested 2024-11-13T18:27:13,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433625 2024-11-13T18:27:13,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433625 2024-11-13T18:27:13,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433625) roll requested 2024-11-13T18:27:13,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433632 2024-11-13T18:27:13,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433632 2024-11-13T18:27:13,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433632) roll requested 2024-11-13T18:27:13,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433640 2024-11-13T18:27:13,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433640 2024-11-13T18:27:13,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433640) roll requested 2024-11-13T18:27:13,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433648 2024-11-13T18:27:13,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433648 2024-11-13T18:27:13,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433648) roll requested 2024-11-13T18:27:13,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433661 2024-11-13T18:27:13,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433661 2024-11-13T18:27:13,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433661) roll requested 2024-11-13T18:27:13,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433670 2024-11-13T18:27:13,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433670 2024-11-13T18:27:13,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433670) roll requested 2024-11-13T18:27:13,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433681 2024-11-13T18:27:13,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433681 2024-11-13T18:27:13,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433681) roll requested 2024-11-13T18:27:13,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433692 2024-11-13T18:27:13,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433692 2024-11-13T18:27:13,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433692) roll requested 2024-11-13T18:27:13,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433699 2024-11-13T18:27:13,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433699 2024-11-13T18:27:13,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433699) roll requested 2024-11-13T18:27:13,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433706 2024-11-13T18:27:13,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433706 2024-11-13T18:27:13,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433706) roll requested 2024-11-13T18:27:13,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433715 2024-11-13T18:27:13,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433715 2024-11-13T18:27:13,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433715) roll requested 2024-11-13T18:27:13,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433723 2024-11-13T18:27:13,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433723 2024-11-13T18:27:13,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433723) roll requested 2024-11-13T18:27:13,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433733 2024-11-13T18:27:13,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433733 2024-11-13T18:27:13,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433733) roll requested 2024-11-13T18:27:13,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433740 2024-11-13T18:27:13,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433740 2024-11-13T18:27:13,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433740) roll requested 2024-11-13T18:27:13,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433747 2024-11-13T18:27:13,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433747 2024-11-13T18:27:13,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433747) roll requested 2024-11-13T18:27:13,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433755 2024-11-13T18:27:13,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433755 2024-11-13T18:27:13,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433755) roll requested 2024-11-13T18:27:13,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433762 2024-11-13T18:27:13,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433762 2024-11-13T18:27:13,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433762) roll requested 2024-11-13T18:27:13,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433770 2024-11-13T18:27:13,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433770 2024-11-13T18:27:13,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433770) roll requested 2024-11-13T18:27:13,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433778 2024-11-13T18:27:13,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433778 2024-11-13T18:27:13,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433778) roll requested 2024-11-13T18:27:13,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433785 2024-11-13T18:27:13,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433785 2024-11-13T18:27:13,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433785) roll requested 2024-11-13T18:27:13,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433794 2024-11-13T18:27:13,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433794 2024-11-13T18:27:13,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433794) roll requested 2024-11-13T18:27:13,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433804 2024-11-13T18:27:13,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433804 2024-11-13T18:27:13,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433804) roll requested 2024-11-13T18:27:13,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433821 2024-11-13T18:27:13,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433821 2024-11-13T18:27:13,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433821) roll requested 2024-11-13T18:27:13,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433829 2024-11-13T18:27:13,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433829 2024-11-13T18:27:13,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433829) roll requested 2024-11-13T18:27:13,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433837 2024-11-13T18:27:13,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433837 2024-11-13T18:27:13,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433837) roll requested 2024-11-13T18:27:13,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433851 2024-11-13T18:27:13,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433851 2024-11-13T18:27:13,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433851) roll requested 2024-11-13T18:27:13,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433861 2024-11-13T18:27:13,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433861 2024-11-13T18:27:13,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433861) roll requested 2024-11-13T18:27:13,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433869 2024-11-13T18:27:13,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433869 2024-11-13T18:27:13,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433869) roll requested 2024-11-13T18:27:13,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433880 2024-11-13T18:27:13,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433880 2024-11-13T18:27:13,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433880) roll requested 2024-11-13T18:27:13,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433888 2024-11-13T18:27:13,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433888 2024-11-13T18:27:13,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433888) roll requested 2024-11-13T18:27:13,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433905 2024-11-13T18:27:13,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433905 2024-11-13T18:27:13,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433905) roll requested 2024-11-13T18:27:13,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433921 2024-11-13T18:27:13,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433921 2024-11-13T18:27:13,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433921) roll requested 2024-11-13T18:27:13,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433941 2024-11-13T18:27:13,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433941 2024-11-13T18:27:13,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433941) roll requested 2024-11-13T18:27:13,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433965 2024-11-13T18:27:13,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433965 2024-11-13T18:27:13,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433965) roll requested 2024-11-13T18:27:13,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433985 2024-11-13T18:27:13,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:13,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433985 2024-11-13T18:27:13,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:13,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:13,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433985) roll requested 2024-11-13T18:27:13,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522433998 2024-11-13T18:27:14,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433998 2024-11-13T18:27:14,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522433998) roll requested 2024-11-13T18:27:14,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434011 2024-11-13T18:27:14,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522433998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434011 2024-11-13T18:27:14,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434011) roll requested 2024-11-13T18:27:14,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434026 2024-11-13T18:27:14,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434026 2024-11-13T18:27:14,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434026) roll requested 2024-11-13T18:27:14,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434041 2024-11-13T18:27:14,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434041 2024-11-13T18:27:14,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434041) roll requested 2024-11-13T18:27:14,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434050 2024-11-13T18:27:14,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434050 2024-11-13T18:27:14,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434050) roll requested 2024-11-13T18:27:14,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434069 2024-11-13T18:27:14,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434069 2024-11-13T18:27:14,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434069) roll requested 2024-11-13T18:27:14,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434081 2024-11-13T18:27:14,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434081 2024-11-13T18:27:14,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434081) roll requested 2024-11-13T18:27:14,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434089 2024-11-13T18:27:14,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434089 2024-11-13T18:27:14,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434089) roll requested 2024-11-13T18:27:14,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434101 2024-11-13T18:27:14,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434101 2024-11-13T18:27:14,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434101) roll requested 2024-11-13T18:27:14,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434113 2024-11-13T18:27:14,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434113 2024-11-13T18:27:14,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434113) roll requested 2024-11-13T18:27:14,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434133 2024-11-13T18:27:14,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434133 2024-11-13T18:27:14,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434133) roll requested 2024-11-13T18:27:14,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434151 2024-11-13T18:27:14,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434151 2024-11-13T18:27:14,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434151) roll requested 2024-11-13T18:27:14,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434165 2024-11-13T18:27:14,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434165 2024-11-13T18:27:14,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434165) roll requested 2024-11-13T18:27:14,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434181 2024-11-13T18:27:14,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434181 2024-11-13T18:27:14,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434181) roll requested 2024-11-13T18:27:14,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434192 2024-11-13T18:27:14,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434192 2024-11-13T18:27:14,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434192) roll requested 2024-11-13T18:27:14,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434200 2024-11-13T18:27:14,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434200 2024-11-13T18:27:14,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434200) roll requested 2024-11-13T18:27:14,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434208 2024-11-13T18:27:14,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434208 2024-11-13T18:27:14,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434208) roll requested 2024-11-13T18:27:14,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434217 2024-11-13T18:27:14,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434217 2024-11-13T18:27:14,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434217) roll requested 2024-11-13T18:27:14,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434225 2024-11-13T18:27:14,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434225 2024-11-13T18:27:14,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434225) roll requested 2024-11-13T18:27:14,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434234 2024-11-13T18:27:14,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434234 2024-11-13T18:27:14,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434234) roll requested 2024-11-13T18:27:14,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434242 2024-11-13T18:27:14,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434242 2024-11-13T18:27:14,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434242) roll requested 2024-11-13T18:27:14,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434251 2024-11-13T18:27:14,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434251 2024-11-13T18:27:14,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434251) roll requested 2024-11-13T18:27:14,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434259 2024-11-13T18:27:14,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434259 2024-11-13T18:27:14,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434259) roll requested 2024-11-13T18:27:14,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434268 2024-11-13T18:27:14,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434268 2024-11-13T18:27:14,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434268) roll requested 2024-11-13T18:27:14,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434275 2024-11-13T18:27:14,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434275 2024-11-13T18:27:14,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434275) roll requested 2024-11-13T18:27:14,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434282 2024-11-13T18:27:14,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434282 2024-11-13T18:27:14,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434282) roll requested 2024-11-13T18:27:14,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434297 2024-11-13T18:27:14,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434297 2024-11-13T18:27:14,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434297) roll requested 2024-11-13T18:27:14,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434317 2024-11-13T18:27:14,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434317 2024-11-13T18:27:14,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434317) roll requested 2024-11-13T18:27:14,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434328 2024-11-13T18:27:14,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434328 2024-11-13T18:27:14,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434328) roll requested 2024-11-13T18:27:14,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434335 2024-11-13T18:27:14,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434335 2024-11-13T18:27:14,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434335) roll requested 2024-11-13T18:27:14,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434344 2024-11-13T18:27:14,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434344 2024-11-13T18:27:14,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434344) roll requested 2024-11-13T18:27:14,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434352 2024-11-13T18:27:14,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434352 2024-11-13T18:27:14,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434352) roll requested 2024-11-13T18:27:14,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434360 2024-11-13T18:27:14,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434360 2024-11-13T18:27:14,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434360) roll requested 2024-11-13T18:27:14,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434369 2024-11-13T18:27:14,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434369 2024-11-13T18:27:14,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434369) roll requested 2024-11-13T18:27:14,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434376 2024-11-13T18:27:14,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434376 2024-11-13T18:27:14,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434376) roll requested 2024-11-13T18:27:14,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434383 2024-11-13T18:27:14,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434383 2024-11-13T18:27:14,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434383) roll requested 2024-11-13T18:27:14,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434391 2024-11-13T18:27:14,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434391 2024-11-13T18:27:14,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434391) roll requested 2024-11-13T18:27:14,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434399 2024-11-13T18:27:14,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434399 2024-11-13T18:27:14,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434399) roll requested 2024-11-13T18:27:14,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434406 2024-11-13T18:27:14,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434406 2024-11-13T18:27:14,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434406) roll requested 2024-11-13T18:27:14,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434414 2024-11-13T18:27:14,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434414 2024-11-13T18:27:14,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434414) roll requested 2024-11-13T18:27:14,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434423 2024-11-13T18:27:14,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434423 2024-11-13T18:27:14,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434423) roll requested 2024-11-13T18:27:14,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434439 2024-11-13T18:27:14,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434439 2024-11-13T18:27:14,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434439) roll requested 2024-11-13T18:27:14,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434453 2024-11-13T18:27:14,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434453 2024-11-13T18:27:14,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434453) roll requested 2024-11-13T18:27:14,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434461 2024-11-13T18:27:14,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434461 2024-11-13T18:27:14,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434461) roll requested 2024-11-13T18:27:14,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434468 2024-11-13T18:27:14,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434468 2024-11-13T18:27:14,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434468) roll requested 2024-11-13T18:27:14,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434476 2024-11-13T18:27:14,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434476 2024-11-13T18:27:14,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434476) roll requested 2024-11-13T18:27:14,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434485 2024-11-13T18:27:14,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434485 2024-11-13T18:27:14,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434485) roll requested 2024-11-13T18:27:14,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434496 2024-11-13T18:27:14,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434496 2024-11-13T18:27:14,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434496) roll requested 2024-11-13T18:27:14,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434505 2024-11-13T18:27:14,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434505 2024-11-13T18:27:14,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434505) roll requested 2024-11-13T18:27:14,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434515 2024-11-13T18:27:14,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434515 2024-11-13T18:27:14,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434515) roll requested 2024-11-13T18:27:14,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434522 2024-11-13T18:27:14,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434522 2024-11-13T18:27:14,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434522) roll requested 2024-11-13T18:27:14,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434528 2024-11-13T18:27:14,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434528 2024-11-13T18:27:14,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434528) roll requested 2024-11-13T18:27:14,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434536 2024-11-13T18:27:14,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434536 2024-11-13T18:27:14,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434536) roll requested 2024-11-13T18:27:14,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434547 2024-11-13T18:27:14,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434547 2024-11-13T18:27:14,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434547) roll requested 2024-11-13T18:27:14,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434556 2024-11-13T18:27:14,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434556 2024-11-13T18:27:14,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434556) roll requested 2024-11-13T18:27:14,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434563 2024-11-13T18:27:14,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434563 2024-11-13T18:27:14,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434563) roll requested 2024-11-13T18:27:14,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434578 2024-11-13T18:27:14,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434578 2024-11-13T18:27:14,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434578) roll requested 2024-11-13T18:27:14,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434587 2024-11-13T18:27:14,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434587 2024-11-13T18:27:14,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434587) roll requested 2024-11-13T18:27:14,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434600 2024-11-13T18:27:14,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434600 2024-11-13T18:27:14,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434600) roll requested 2024-11-13T18:27:14,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434607 2024-11-13T18:27:14,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434607 2024-11-13T18:27:14,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434607) roll requested 2024-11-13T18:27:14,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434627 2024-11-13T18:27:14,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434627 2024-11-13T18:27:14,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434627) roll requested 2024-11-13T18:27:14,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434638 2024-11-13T18:27:14,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434638 2024-11-13T18:27:14,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434638) roll requested 2024-11-13T18:27:14,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434648 2024-11-13T18:27:14,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434648 2024-11-13T18:27:14,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434648) roll requested 2024-11-13T18:27:14,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434665 2024-11-13T18:27:14,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434665 2024-11-13T18:27:14,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434665) roll requested 2024-11-13T18:27:14,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434673 2024-11-13T18:27:14,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434673 2024-11-13T18:27:14,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434673) roll requested 2024-11-13T18:27:14,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434681 2024-11-13T18:27:14,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434681 2024-11-13T18:27:14,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434681) roll requested 2024-11-13T18:27:14,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434689 2024-11-13T18:27:14,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434689 2024-11-13T18:27:14,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434689) roll requested 2024-11-13T18:27:14,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434696 2024-11-13T18:27:14,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434696 2024-11-13T18:27:14,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434696) roll requested 2024-11-13T18:27:14,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434704 2024-11-13T18:27:14,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434704 2024-11-13T18:27:14,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434704) roll requested 2024-11-13T18:27:14,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434712 2024-11-13T18:27:14,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434712 2024-11-13T18:27:14,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434712) roll requested 2024-11-13T18:27:14,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434727 2024-11-13T18:27:14,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434727 2024-11-13T18:27:14,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434727) roll requested 2024-11-13T18:27:14,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434737 2024-11-13T18:27:14,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434737 2024-11-13T18:27:14,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434737) roll requested 2024-11-13T18:27:14,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434744 2024-11-13T18:27:14,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434744 2024-11-13T18:27:14,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434744) roll requested 2024-11-13T18:27:14,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434755 2024-11-13T18:27:14,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434755 2024-11-13T18:27:14,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434755) roll requested 2024-11-13T18:27:14,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434765 2024-11-13T18:27:14,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434765 2024-11-13T18:27:14,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434765) roll requested 2024-11-13T18:27:14,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434777 2024-11-13T18:27:14,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434777 2024-11-13T18:27:14,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434777) roll requested 2024-11-13T18:27:14,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434793 2024-11-13T18:27:14,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434793 2024-11-13T18:27:14,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434793) roll requested 2024-11-13T18:27:14,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434802 2024-11-13T18:27:14,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434802 2024-11-13T18:27:14,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434802) roll requested 2024-11-13T18:27:14,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434809 2024-11-13T18:27:14,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434809 2024-11-13T18:27:14,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434809) roll requested 2024-11-13T18:27:14,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434816 2024-11-13T18:27:14,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434816 2024-11-13T18:27:14,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434816) roll requested 2024-11-13T18:27:14,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434827 2024-11-13T18:27:14,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434827 2024-11-13T18:27:14,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434827) roll requested 2024-11-13T18:27:14,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434835 2024-11-13T18:27:14,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434835 2024-11-13T18:27:14,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434835) roll requested 2024-11-13T18:27:14,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434846 2024-11-13T18:27:14,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434846 2024-11-13T18:27:14,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434846) roll requested 2024-11-13T18:27:14,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434855 2024-11-13T18:27:14,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434855 2024-11-13T18:27:14,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434855) roll requested 2024-11-13T18:27:14,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434862 2024-11-13T18:27:14,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434862 2024-11-13T18:27:14,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434862) roll requested 2024-11-13T18:27:14,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434870 2024-11-13T18:27:14,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434870 2024-11-13T18:27:14,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434870) roll requested 2024-11-13T18:27:14,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434877 2024-11-13T18:27:14,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434877 2024-11-13T18:27:14,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434877) roll requested 2024-11-13T18:27:14,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434885 2024-11-13T18:27:14,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434885 2024-11-13T18:27:14,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434885) roll requested 2024-11-13T18:27:14,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434892 2024-11-13T18:27:14,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434892 2024-11-13T18:27:14,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434892) roll requested 2024-11-13T18:27:14,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434900 2024-11-13T18:27:14,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434900 2024-11-13T18:27:14,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434900) roll requested 2024-11-13T18:27:14,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434908 2024-11-13T18:27:14,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434908 2024-11-13T18:27:14,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434908) roll requested 2024-11-13T18:27:14,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434916 2024-11-13T18:27:14,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434916 2024-11-13T18:27:14,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434916) roll requested 2024-11-13T18:27:14,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434935 2024-11-13T18:27:14,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434935 2024-11-13T18:27:14,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434935) roll requested 2024-11-13T18:27:14,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434952 2024-11-13T18:27:14,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434952 2024-11-13T18:27:14,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434952) roll requested 2024-11-13T18:27:14,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434963 2024-11-13T18:27:14,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434963 2024-11-13T18:27:14,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434963) roll requested 2024-11-13T18:27:14,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434981 2024-11-13T18:27:14,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:14,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434981 2024-11-13T18:27:14,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:14,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:14,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434981) roll requested 2024-11-13T18:27:14,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522434998 2024-11-13T18:27:15,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434998 2024-11-13T18:27:15,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522434998) roll requested 2024-11-13T18:27:15,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435009 2024-11-13T18:27:15,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522434998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435009 2024-11-13T18:27:15,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435009) roll requested 2024-11-13T18:27:15,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435021 2024-11-13T18:27:15,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435021 2024-11-13T18:27:15,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435021) roll requested 2024-11-13T18:27:15,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435037 2024-11-13T18:27:15,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435037 2024-11-13T18:27:15,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435037) roll requested 2024-11-13T18:27:15,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435047 2024-11-13T18:27:15,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435047 2024-11-13T18:27:15,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435047) roll requested 2024-11-13T18:27:15,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435055 2024-11-13T18:27:15,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435055 2024-11-13T18:27:15,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435055) roll requested 2024-11-13T18:27:15,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435064 2024-11-13T18:27:15,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435064 2024-11-13T18:27:15,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435064) roll requested 2024-11-13T18:27:15,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435073 2024-11-13T18:27:15,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435073 2024-11-13T18:27:15,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435073) roll requested 2024-11-13T18:27:15,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435080 2024-11-13T18:27:15,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435080 2024-11-13T18:27:15,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435080) roll requested 2024-11-13T18:27:15,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435087 2024-11-13T18:27:15,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435087 2024-11-13T18:27:15,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435087) roll requested 2024-11-13T18:27:15,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435095 2024-11-13T18:27:15,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435095 2024-11-13T18:27:15,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435095) roll requested 2024-11-13T18:27:15,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435102 2024-11-13T18:27:15,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435102 2024-11-13T18:27:15,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435102) roll requested 2024-11-13T18:27:15,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435109 2024-11-13T18:27:15,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435109 2024-11-13T18:27:15,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435109) roll requested 2024-11-13T18:27:15,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435116 2024-11-13T18:27:15,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435116 2024-11-13T18:27:15,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435116) roll requested 2024-11-13T18:27:15,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435132 2024-11-13T18:27:15,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435132 2024-11-13T18:27:15,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435132) roll requested 2024-11-13T18:27:15,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435145 2024-11-13T18:27:15,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435145 2024-11-13T18:27:15,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435145) roll requested 2024-11-13T18:27:15,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435156 2024-11-13T18:27:15,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435156 2024-11-13T18:27:15,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435156) roll requested 2024-11-13T18:27:15,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435165 2024-11-13T18:27:15,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435165 2024-11-13T18:27:15,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435165) roll requested 2024-11-13T18:27:15,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435174 2024-11-13T18:27:15,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435174 2024-11-13T18:27:15,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435174) roll requested 2024-11-13T18:27:15,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435193 2024-11-13T18:27:15,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435193 2024-11-13T18:27:15,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435193) roll requested 2024-11-13T18:27:15,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435213 2024-11-13T18:27:15,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435213 2024-11-13T18:27:15,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435213) roll requested 2024-11-13T18:27:15,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435225 2024-11-13T18:27:15,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435225 2024-11-13T18:27:15,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435225) roll requested 2024-11-13T18:27:15,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435237 2024-11-13T18:27:15,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435237 2024-11-13T18:27:15,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435237) roll requested 2024-11-13T18:27:15,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435245 2024-11-13T18:27:15,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435245 2024-11-13T18:27:15,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435245) roll requested 2024-11-13T18:27:15,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435255 2024-11-13T18:27:15,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435255 2024-11-13T18:27:15,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435255) roll requested 2024-11-13T18:27:15,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435264 2024-11-13T18:27:15,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435264 2024-11-13T18:27:15,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435264) roll requested 2024-11-13T18:27:15,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435272 2024-11-13T18:27:15,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435272 2024-11-13T18:27:15,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435272) roll requested 2024-11-13T18:27:15,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435279 2024-11-13T18:27:15,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435279 2024-11-13T18:27:15,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435279) roll requested 2024-11-13T18:27:15,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435287 2024-11-13T18:27:15,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435287 2024-11-13T18:27:15,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435287) roll requested 2024-11-13T18:27:15,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435301 2024-11-13T18:27:15,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435301 2024-11-13T18:27:15,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435301) roll requested 2024-11-13T18:27:15,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435310 2024-11-13T18:27:15,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435310 2024-11-13T18:27:15,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435310) roll requested 2024-11-13T18:27:15,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435319 2024-11-13T18:27:15,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435319 2024-11-13T18:27:15,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435319) roll requested 2024-11-13T18:27:15,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435328 2024-11-13T18:27:15,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435328 2024-11-13T18:27:15,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435328) roll requested 2024-11-13T18:27:15,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435336 2024-11-13T18:27:15,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435336 2024-11-13T18:27:15,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435336) roll requested 2024-11-13T18:27:15,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435343 2024-11-13T18:27:15,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435343 2024-11-13T18:27:15,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435343) roll requested 2024-11-13T18:27:15,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435352 2024-11-13T18:27:15,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435352 2024-11-13T18:27:15,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435352) roll requested 2024-11-13T18:27:15,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435359 2024-11-13T18:27:15,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435359 2024-11-13T18:27:15,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435359) roll requested 2024-11-13T18:27:15,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435366 2024-11-13T18:27:15,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435366 2024-11-13T18:27:15,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435366) roll requested 2024-11-13T18:27:15,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435375 2024-11-13T18:27:15,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435375 2024-11-13T18:27:15,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435375) roll requested 2024-11-13T18:27:15,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435382 2024-11-13T18:27:15,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435382 2024-11-13T18:27:15,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435382) roll requested 2024-11-13T18:27:15,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435390 2024-11-13T18:27:15,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435390 2024-11-13T18:27:15,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435390) roll requested 2024-11-13T18:27:15,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435398 2024-11-13T18:27:15,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435398 2024-11-13T18:27:15,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435398) roll requested 2024-11-13T18:27:15,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435405 2024-11-13T18:27:15,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435405 2024-11-13T18:27:15,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435405) roll requested 2024-11-13T18:27:15,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435414 2024-11-13T18:27:15,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435414 2024-11-13T18:27:15,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435414) roll requested 2024-11-13T18:27:15,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435422 2024-11-13T18:27:15,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435422 2024-11-13T18:27:15,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435422) roll requested 2024-11-13T18:27:15,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435430 2024-11-13T18:27:15,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435430 2024-11-13T18:27:15,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435430) roll requested 2024-11-13T18:27:15,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435440 2024-11-13T18:27:15,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435440 2024-11-13T18:27:15,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435440) roll requested 2024-11-13T18:27:15,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435448 2024-11-13T18:27:15,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435448 2024-11-13T18:27:15,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435448) roll requested 2024-11-13T18:27:15,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435456 2024-11-13T18:27:15,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435456 2024-11-13T18:27:15,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435456) roll requested 2024-11-13T18:27:15,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435466 2024-11-13T18:27:15,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435466 2024-11-13T18:27:15,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435466) roll requested 2024-11-13T18:27:15,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435474 2024-11-13T18:27:15,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435474 2024-11-13T18:27:15,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435474) roll requested 2024-11-13T18:27:15,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435488 2024-11-13T18:27:15,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435488 2024-11-13T18:27:15,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435488) roll requested 2024-11-13T18:27:15,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435501 2024-11-13T18:27:15,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435501 2024-11-13T18:27:15,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435501) roll requested 2024-11-13T18:27:15,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435513 2024-11-13T18:27:15,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435513 2024-11-13T18:27:15,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435513) roll requested 2024-11-13T18:27:15,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435522 2024-11-13T18:27:15,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435522 2024-11-13T18:27:15,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435522) roll requested 2024-11-13T18:27:15,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435529 2024-11-13T18:27:15,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435529 2024-11-13T18:27:15,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435529) roll requested 2024-11-13T18:27:15,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435537 2024-11-13T18:27:15,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435537 2024-11-13T18:27:15,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435537) roll requested 2024-11-13T18:27:15,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435544 2024-11-13T18:27:15,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435544 2024-11-13T18:27:15,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435544) roll requested 2024-11-13T18:27:15,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435557 2024-11-13T18:27:15,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435557 2024-11-13T18:27:15,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435557) roll requested 2024-11-13T18:27:15,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435563 2024-11-13T18:27:15,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435563 2024-11-13T18:27:15,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435563) roll requested 2024-11-13T18:27:15,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435571 2024-11-13T18:27:15,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435571 2024-11-13T18:27:15,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435571) roll requested 2024-11-13T18:27:15,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435581 2024-11-13T18:27:15,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435581 2024-11-13T18:27:15,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435581) roll requested 2024-11-13T18:27:15,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435588 2024-11-13T18:27:15,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435588 2024-11-13T18:27:15,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435588) roll requested 2024-11-13T18:27:15,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435596 2024-11-13T18:27:15,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435596 2024-11-13T18:27:15,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435596) roll requested 2024-11-13T18:27:15,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435603 2024-11-13T18:27:15,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435603 2024-11-13T18:27:15,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435603) roll requested 2024-11-13T18:27:15,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435617 2024-11-13T18:27:15,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435617 2024-11-13T18:27:15,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435617) roll requested 2024-11-13T18:27:15,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435625 2024-11-13T18:27:15,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435625 2024-11-13T18:27:15,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435625) roll requested 2024-11-13T18:27:15,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435633 2024-11-13T18:27:15,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435633 2024-11-13T18:27:15,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435633) roll requested 2024-11-13T18:27:15,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435646 2024-11-13T18:27:15,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435646 2024-11-13T18:27:15,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435646) roll requested 2024-11-13T18:27:15,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435657 2024-11-13T18:27:15,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435657 2024-11-13T18:27:15,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435657) roll requested 2024-11-13T18:27:15,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435665 2024-11-13T18:27:15,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435665 2024-11-13T18:27:15,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435665) roll requested 2024-11-13T18:27:15,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435673 2024-11-13T18:27:15,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435673 2024-11-13T18:27:15,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435673) roll requested 2024-11-13T18:27:15,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435681 2024-11-13T18:27:15,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435681 2024-11-13T18:27:15,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435681) roll requested 2024-11-13T18:27:15,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435689 2024-11-13T18:27:15,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435689 2024-11-13T18:27:15,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435689) roll requested 2024-11-13T18:27:15,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435705 2024-11-13T18:27:15,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435705 2024-11-13T18:27:15,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435705) roll requested 2024-11-13T18:27:15,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435713 2024-11-13T18:27:15,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435713 2024-11-13T18:27:15,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435713) roll requested 2024-11-13T18:27:15,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435729 2024-11-13T18:27:15,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435729 2024-11-13T18:27:15,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435729) roll requested 2024-11-13T18:27:15,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435741 2024-11-13T18:27:15,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435741 2024-11-13T18:27:15,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435741) roll requested 2024-11-13T18:27:15,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435748 2024-11-13T18:27:15,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435748 2024-11-13T18:27:15,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435748) roll requested 2024-11-13T18:27:15,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435765 2024-11-13T18:27:15,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435765 2024-11-13T18:27:15,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435765) roll requested 2024-11-13T18:27:15,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435772 2024-11-13T18:27:15,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435772 2024-11-13T18:27:15,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435772) roll requested 2024-11-13T18:27:15,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435782 2024-11-13T18:27:15,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435782 2024-11-13T18:27:15,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435782) roll requested 2024-11-13T18:27:15,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435795 2024-11-13T18:27:15,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435795 2024-11-13T18:27:15,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435795) roll requested 2024-11-13T18:27:15,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435803 2024-11-13T18:27:15,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435803 2024-11-13T18:27:15,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435803) roll requested 2024-11-13T18:27:15,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435816 2024-11-13T18:27:15,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435816 2024-11-13T18:27:15,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435816) roll requested 2024-11-13T18:27:15,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435833 2024-11-13T18:27:15,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435833 2024-11-13T18:27:15,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435833) roll requested 2024-11-13T18:27:15,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435849 2024-11-13T18:27:15,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435849 2024-11-13T18:27:15,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435849) roll requested 2024-11-13T18:27:15,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435866 2024-11-13T18:27:15,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435866 2024-11-13T18:27:15,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435849 is not closed yet, will try archiving it next time 2024-11-13T18:27:15,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435866) roll requested 2024-11-13T18:27:15,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435880 2024-11-13T18:27:15,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435880 2024-11-13T18:27:15,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435866 is not closed yet, will try archiving it next time 2024-11-13T18:27:15,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435880) roll requested 2024-11-13T18:27:15,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435891 2024-11-13T18:27:15,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435891 2024-11-13T18:27:15,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435891) roll requested 2024-11-13T18:27:15,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435902 2024-11-13T18:27:15,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435902 2024-11-13T18:27:15,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435902) roll requested 2024-11-13T18:27:15,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435913 2024-11-13T18:27:15,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435913 2024-11-13T18:27:15,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435913) roll requested 2024-11-13T18:27:15,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435941 2024-11-13T18:27:15,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435941 2024-11-13T18:27:15,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435941) roll requested 2024-11-13T18:27:15,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435959 2024-11-13T18:27:15,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435959 2024-11-13T18:27:15,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435959) roll requested 2024-11-13T18:27:15,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435973 2024-11-13T18:27:15,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:15,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435973 2024-11-13T18:27:15,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:15,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:15,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435973) roll requested 2024-11-13T18:27:15,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522435993 2024-11-13T18:27:16,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435993 2024-11-13T18:27:16,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522435993) roll requested 2024-11-13T18:27:16,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436008 2024-11-13T18:27:16,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522435993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436008 2024-11-13T18:27:16,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436008) roll requested 2024-11-13T18:27:16,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436027 2024-11-13T18:27:16,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436027 2024-11-13T18:27:16,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436027) roll requested 2024-11-13T18:27:16,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436065 2024-11-13T18:27:16,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436065 2024-11-13T18:27:16,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436065) roll requested 2024-11-13T18:27:16,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436101 2024-11-13T18:27:16,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436101 2024-11-13T18:27:16,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436101) roll requested 2024-11-13T18:27:16,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436126 2024-11-13T18:27:16,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436126 2024-11-13T18:27:16,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436126) roll requested 2024-11-13T18:27:16,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436141 2024-11-13T18:27:16,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436141 2024-11-13T18:27:16,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436141) roll requested 2024-11-13T18:27:16,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436153 2024-11-13T18:27:16,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436153 2024-11-13T18:27:16,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436153) roll requested 2024-11-13T18:27:16,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436169 2024-11-13T18:27:16,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436169 2024-11-13T18:27:16,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436169) roll requested 2024-11-13T18:27:16,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436184 2024-11-13T18:27:16,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436184 2024-11-13T18:27:16,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436184) roll requested 2024-11-13T18:27:16,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436197 2024-11-13T18:27:16,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436197 2024-11-13T18:27:16,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436197) roll requested 2024-11-13T18:27:16,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436215 2024-11-13T18:27:16,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436215 2024-11-13T18:27:16,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436215) roll requested 2024-11-13T18:27:16,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436225 2024-11-13T18:27:16,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436225 2024-11-13T18:27:16,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436225) roll requested 2024-11-13T18:27:16,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436241 2024-11-13T18:27:16,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436241 2024-11-13T18:27:16,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436241) roll requested 2024-11-13T18:27:16,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436249 2024-11-13T18:27:16,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436249 2024-11-13T18:27:16,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436249) roll requested 2024-11-13T18:27:16,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436264 2024-11-13T18:27:16,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436264 2024-11-13T18:27:16,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436264) roll requested 2024-11-13T18:27:16,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436277 2024-11-13T18:27:16,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436277 2024-11-13T18:27:16,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436277) roll requested 2024-11-13T18:27:16,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436293 2024-11-13T18:27:16,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436293 2024-11-13T18:27:16,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436293) roll requested 2024-11-13T18:27:16,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436301 2024-11-13T18:27:16,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436301 2024-11-13T18:27:16,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436301) roll requested 2024-11-13T18:27:16,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436310 2024-11-13T18:27:16,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436310 2024-11-13T18:27:16,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436310) roll requested 2024-11-13T18:27:16,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436317 2024-11-13T18:27:16,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436317 2024-11-13T18:27:16,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436317) roll requested 2024-11-13T18:27:16,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436328 2024-11-13T18:27:16,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436328 2024-11-13T18:27:16,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436328) roll requested 2024-11-13T18:27:16,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436338 2024-11-13T18:27:16,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436338 2024-11-13T18:27:16,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436338) roll requested 2024-11-13T18:27:16,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436345 2024-11-13T18:27:16,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436345 2024-11-13T18:27:16,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436345) roll requested 2024-11-13T18:27:16,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436358 2024-11-13T18:27:16,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436358 2024-11-13T18:27:16,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436358) roll requested 2024-11-13T18:27:16,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436366 2024-11-13T18:27:16,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436366 2024-11-13T18:27:16,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436366) roll requested 2024-11-13T18:27:16,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436374 2024-11-13T18:27:16,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436374 2024-11-13T18:27:16,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436374) roll requested 2024-11-13T18:27:16,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436385 2024-11-13T18:27:16,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436385 2024-11-13T18:27:16,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436385) roll requested 2024-11-13T18:27:16,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436393 2024-11-13T18:27:16,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436393 2024-11-13T18:27:16,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436393) roll requested 2024-11-13T18:27:16,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436401 2024-11-13T18:27:16,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436401 2024-11-13T18:27:16,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436401) roll requested 2024-11-13T18:27:16,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436409 2024-11-13T18:27:16,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436409 2024-11-13T18:27:16,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436409) roll requested 2024-11-13T18:27:16,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436415 2024-11-13T18:27:16,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436415 2024-11-13T18:27:16,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436415) roll requested 2024-11-13T18:27:16,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436423 2024-11-13T18:27:16,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436423 2024-11-13T18:27:16,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436423) roll requested 2024-11-13T18:27:16,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436431 2024-11-13T18:27:16,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436431 2024-11-13T18:27:16,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436431) roll requested 2024-11-13T18:27:16,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436439 2024-11-13T18:27:16,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436439 2024-11-13T18:27:16,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436439) roll requested 2024-11-13T18:27:16,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436457 2024-11-13T18:27:16,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436457 2024-11-13T18:27:16,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436457) roll requested 2024-11-13T18:27:16,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436468 2024-11-13T18:27:16,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436468 2024-11-13T18:27:16,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436468) roll requested 2024-11-13T18:27:16,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436476 2024-11-13T18:27:16,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436476 2024-11-13T18:27:16,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436476) roll requested 2024-11-13T18:27:16,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436484 2024-11-13T18:27:16,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436484 2024-11-13T18:27:16,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436484) roll requested 2024-11-13T18:27:16,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436490 2024-11-13T18:27:16,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436490 2024-11-13T18:27:16,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436490) roll requested 2024-11-13T18:27:16,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436498 2024-11-13T18:27:16,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436498 2024-11-13T18:27:16,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436498) roll requested 2024-11-13T18:27:16,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436506 2024-11-13T18:27:16,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436506 2024-11-13T18:27:16,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436506) roll requested 2024-11-13T18:27:16,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436514 2024-11-13T18:27:16,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436514 2024-11-13T18:27:16,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436514) roll requested 2024-11-13T18:27:16,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436521 2024-11-13T18:27:16,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436521 2024-11-13T18:27:16,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436521) roll requested 2024-11-13T18:27:16,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436527 2024-11-13T18:27:16,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436527 2024-11-13T18:27:16,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436527) roll requested 2024-11-13T18:27:16,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436535 2024-11-13T18:27:16,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436535 2024-11-13T18:27:16,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436535) roll requested 2024-11-13T18:27:16,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436543 2024-11-13T18:27:16,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436543 2024-11-13T18:27:16,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436543) roll requested 2024-11-13T18:27:16,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436551 2024-11-13T18:27:16,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436551 2024-11-13T18:27:16,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436551) roll requested 2024-11-13T18:27:16,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436559 2024-11-13T18:27:16,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436559 2024-11-13T18:27:16,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436559) roll requested 2024-11-13T18:27:16,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436567 2024-11-13T18:27:16,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436567 2024-11-13T18:27:16,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436567) roll requested 2024-11-13T18:27:16,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436575 2024-11-13T18:27:16,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436575 2024-11-13T18:27:16,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436575) roll requested 2024-11-13T18:27:16,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436582 2024-11-13T18:27:16,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436582 2024-11-13T18:27:16,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436582) roll requested 2024-11-13T18:27:16,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436593 2024-11-13T18:27:16,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436593 2024-11-13T18:27:16,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436593) roll requested 2024-11-13T18:27:16,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436605 2024-11-13T18:27:16,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436605 2024-11-13T18:27:16,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436605) roll requested 2024-11-13T18:27:16,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436618 2024-11-13T18:27:16,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436618 2024-11-13T18:27:16,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436618) roll requested 2024-11-13T18:27:16,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436626 2024-11-13T18:27:16,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436626 2024-11-13T18:27:16,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436626) roll requested 2024-11-13T18:27:16,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436634 2024-11-13T18:27:16,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436634 2024-11-13T18:27:16,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436634) roll requested 2024-11-13T18:27:16,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436643 2024-11-13T18:27:16,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436643 2024-11-13T18:27:16,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436643) roll requested 2024-11-13T18:27:16,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436650 2024-11-13T18:27:16,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436650 2024-11-13T18:27:16,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436650) roll requested 2024-11-13T18:27:16,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436664 2024-11-13T18:27:16,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436664 2024-11-13T18:27:16,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436664) roll requested 2024-11-13T18:27:16,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436671 2024-11-13T18:27:16,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436671 2024-11-13T18:27:16,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436671) roll requested 2024-11-13T18:27:16,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436681 2024-11-13T18:27:16,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436681 2024-11-13T18:27:16,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436681) roll requested 2024-11-13T18:27:16,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436689 2024-11-13T18:27:16,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436689 2024-11-13T18:27:16,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436689) roll requested 2024-11-13T18:27:16,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436696 2024-11-13T18:27:16,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436696 2024-11-13T18:27:16,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436696) roll requested 2024-11-13T18:27:16,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436705 2024-11-13T18:27:16,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436705 2024-11-13T18:27:16,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436705) roll requested 2024-11-13T18:27:16,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436713 2024-11-13T18:27:16,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436713 2024-11-13T18:27:16,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436713) roll requested 2024-11-13T18:27:16,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436721 2024-11-13T18:27:16,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436721 2024-11-13T18:27:16,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436721) roll requested 2024-11-13T18:27:16,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436733 2024-11-13T18:27:16,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436733 2024-11-13T18:27:16,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436733) roll requested 2024-11-13T18:27:16,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436742 2024-11-13T18:27:16,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436742 2024-11-13T18:27:16,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436742) roll requested 2024-11-13T18:27:16,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436750 2024-11-13T18:27:16,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436750 2024-11-13T18:27:16,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436750) roll requested 2024-11-13T18:27:16,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436757 2024-11-13T18:27:16,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436757 2024-11-13T18:27:16,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436757) roll requested 2024-11-13T18:27:16,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436764 2024-11-13T18:27:16,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436764 2024-11-13T18:27:16,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436764) roll requested 2024-11-13T18:27:16,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436771 2024-11-13T18:27:16,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436771 2024-11-13T18:27:16,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436771) roll requested 2024-11-13T18:27:16,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436779 2024-11-13T18:27:16,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436779 2024-11-13T18:27:16,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436779) roll requested 2024-11-13T18:27:16,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436786 2024-11-13T18:27:16,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436786 2024-11-13T18:27:16,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436786) roll requested 2024-11-13T18:27:16,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436794 2024-11-13T18:27:16,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436794 2024-11-13T18:27:16,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436794) roll requested 2024-11-13T18:27:16,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436805 2024-11-13T18:27:16,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436805 2024-11-13T18:27:16,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436805) roll requested 2024-11-13T18:27:16,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436812 2024-11-13T18:27:16,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436812 2024-11-13T18:27:16,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436812) roll requested 2024-11-13T18:27:16,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436820 2024-11-13T18:27:16,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436820 2024-11-13T18:27:16,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436820) roll requested 2024-11-13T18:27:16,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436828 2024-11-13T18:27:16,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436828 2024-11-13T18:27:16,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436828) roll requested 2024-11-13T18:27:16,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436836 2024-11-13T18:27:16,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436836 2024-11-13T18:27:16,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436836) roll requested 2024-11-13T18:27:16,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436852 2024-11-13T18:27:16,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436852 2024-11-13T18:27:16,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436852) roll requested 2024-11-13T18:27:16,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436861 2024-11-13T18:27:16,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436861 2024-11-13T18:27:16,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436861) roll requested 2024-11-13T18:27:16,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436868 2024-11-13T18:27:16,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436868 2024-11-13T18:27:16,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436868) roll requested 2024-11-13T18:27:16,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436877 2024-11-13T18:27:16,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436877 2024-11-13T18:27:16,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436877) roll requested 2024-11-13T18:27:16,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436885 2024-11-13T18:27:16,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436885 2024-11-13T18:27:16,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436885) roll requested 2024-11-13T18:27:16,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436893 2024-11-13T18:27:16,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436893 2024-11-13T18:27:16,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436893) roll requested 2024-11-13T18:27:16,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436901 2024-11-13T18:27:16,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436901 2024-11-13T18:27:16,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436901) roll requested 2024-11-13T18:27:16,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436911 2024-11-13T18:27:16,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436911 2024-11-13T18:27:16,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436911) roll requested 2024-11-13T18:27:16,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436918 2024-11-13T18:27:16,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436918 2024-11-13T18:27:16,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436918) roll requested 2024-11-13T18:27:16,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436925 2024-11-13T18:27:16,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436925 2024-11-13T18:27:16,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436925) roll requested 2024-11-13T18:27:16,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436933 2024-11-13T18:27:16,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436933 2024-11-13T18:27:16,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436933) roll requested 2024-11-13T18:27:16,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436946 2024-11-13T18:27:16,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436946 2024-11-13T18:27:16,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436946) roll requested 2024-11-13T18:27:16,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436956 2024-11-13T18:27:16,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436956 2024-11-13T18:27:16,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436956) roll requested 2024-11-13T18:27:16,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436964 2024-11-13T18:27:16,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436964 2024-11-13T18:27:16,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436964) roll requested 2024-11-13T18:27:16,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436972 2024-11-13T18:27:16,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436972 2024-11-13T18:27:16,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436972) roll requested 2024-11-13T18:27:16,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436979 2024-11-13T18:27:16,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436979 2024-11-13T18:27:16,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436979) roll requested 2024-11-13T18:27:16,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436988 2024-11-13T18:27:16,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:16,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436988 2024-11-13T18:27:16,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:16,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:16,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436988) roll requested 2024-11-13T18:27:16,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522436997 2024-11-13T18:27:17,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436997 2024-11-13T18:27:17,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522436997) roll requested 2024-11-13T18:27:17,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437003 2024-11-13T18:27:17,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522436997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437003 2024-11-13T18:27:17,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437003) roll requested 2024-11-13T18:27:17,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437011 2024-11-13T18:27:17,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437011 2024-11-13T18:27:17,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437011) roll requested 2024-11-13T18:27:17,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437018 2024-11-13T18:27:17,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437018 2024-11-13T18:27:17,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437018) roll requested 2024-11-13T18:27:17,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437026 2024-11-13T18:27:17,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437026 2024-11-13T18:27:17,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437026) roll requested 2024-11-13T18:27:17,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437035 2024-11-13T18:27:17,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437035 2024-11-13T18:27:17,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437035) roll requested 2024-11-13T18:27:17,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437042 2024-11-13T18:27:17,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437042 2024-11-13T18:27:17,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437042) roll requested 2024-11-13T18:27:17,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437049 2024-11-13T18:27:17,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437049 2024-11-13T18:27:17,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437049) roll requested 2024-11-13T18:27:17,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437057 2024-11-13T18:27:17,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437057 2024-11-13T18:27:17,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437057) roll requested 2024-11-13T18:27:17,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437065 2024-11-13T18:27:17,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437065 2024-11-13T18:27:17,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437065) roll requested 2024-11-13T18:27:17,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437073 2024-11-13T18:27:17,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437073 2024-11-13T18:27:17,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437073) roll requested 2024-11-13T18:27:17,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437080 2024-11-13T18:27:17,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437080 2024-11-13T18:27:17,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437080) roll requested 2024-11-13T18:27:17,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437088 2024-11-13T18:27:17,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437088 2024-11-13T18:27:17,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437088) roll requested 2024-11-13T18:27:17,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437096 2024-11-13T18:27:17,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437096 2024-11-13T18:27:17,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437096) roll requested 2024-11-13T18:27:17,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437104 2024-11-13T18:27:17,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437104 2024-11-13T18:27:17,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437104) roll requested 2024-11-13T18:27:17,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437113 2024-11-13T18:27:17,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437113 2024-11-13T18:27:17,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437113) roll requested 2024-11-13T18:27:17,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437129 2024-11-13T18:27:17,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437129 2024-11-13T18:27:17,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437129) roll requested 2024-11-13T18:27:17,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437136 2024-11-13T18:27:17,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437136 2024-11-13T18:27:17,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437136) roll requested 2024-11-13T18:27:17,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437148 2024-11-13T18:27:17,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437148 2024-11-13T18:27:17,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437148) roll requested 2024-11-13T18:27:17,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437154 2024-11-13T18:27:17,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437154 2024-11-13T18:27:17,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437154) roll requested 2024-11-13T18:27:17,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437162 2024-11-13T18:27:17,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437162 2024-11-13T18:27:17,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437162) roll requested 2024-11-13T18:27:17,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437169 2024-11-13T18:27:17,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437169 2024-11-13T18:27:17,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437169) roll requested 2024-11-13T18:27:17,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437177 2024-11-13T18:27:17,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437177 2024-11-13T18:27:17,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437177) roll requested 2024-11-13T18:27:17,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437193 2024-11-13T18:27:17,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437193 2024-11-13T18:27:17,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437193) roll requested 2024-11-13T18:27:17,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437210 2024-11-13T18:27:17,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437210 2024-11-13T18:27:17,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437210) roll requested 2024-11-13T18:27:17,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437221 2024-11-13T18:27:17,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437221 2024-11-13T18:27:17,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437221) roll requested 2024-11-13T18:27:17,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437229 2024-11-13T18:27:17,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437229 2024-11-13T18:27:17,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437229) roll requested 2024-11-13T18:27:17,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437237 2024-11-13T18:27:17,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437237 2024-11-13T18:27:17,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437237) roll requested 2024-11-13T18:27:17,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437245 2024-11-13T18:27:17,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437245 2024-11-13T18:27:17,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437245) roll requested 2024-11-13T18:27:17,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437253 2024-11-13T18:27:17,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437253 2024-11-13T18:27:17,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437253) roll requested 2024-11-13T18:27:17,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437262 2024-11-13T18:27:17,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437262 2024-11-13T18:27:17,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437262) roll requested 2024-11-13T18:27:17,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437269 2024-11-13T18:27:17,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437269 2024-11-13T18:27:17,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437269) roll requested 2024-11-13T18:27:17,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437279 2024-11-13T18:27:17,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437279 2024-11-13T18:27:17,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437279) roll requested 2024-11-13T18:27:17,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437289 2024-11-13T18:27:17,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437289 2024-11-13T18:27:17,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437289) roll requested 2024-11-13T18:27:17,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437296 2024-11-13T18:27:17,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437296 2024-11-13T18:27:17,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437296) roll requested 2024-11-13T18:27:17,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437304 2024-11-13T18:27:17,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437304 2024-11-13T18:27:17,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437304) roll requested 2024-11-13T18:27:17,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437311 2024-11-13T18:27:17,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437311 2024-11-13T18:27:17,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437311) roll requested 2024-11-13T18:27:17,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437318 2024-11-13T18:27:17,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437318 2024-11-13T18:27:17,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437318) roll requested 2024-11-13T18:27:17,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437324 2024-11-13T18:27:17,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437324 2024-11-13T18:27:17,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437324) roll requested 2024-11-13T18:27:17,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437334 2024-11-13T18:27:17,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437334 2024-11-13T18:27:17,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437334) roll requested 2024-11-13T18:27:17,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437342 2024-11-13T18:27:17,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437342 2024-11-13T18:27:17,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437342) roll requested 2024-11-13T18:27:17,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437349 2024-11-13T18:27:17,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437349 2024-11-13T18:27:17,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437349) roll requested 2024-11-13T18:27:17,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437362 2024-11-13T18:27:17,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437362 2024-11-13T18:27:17,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437362) roll requested 2024-11-13T18:27:17,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437373 2024-11-13T18:27:17,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437373 2024-11-13T18:27:17,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437373) roll requested 2024-11-13T18:27:17,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437396 2024-11-13T18:27:17,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437396 2024-11-13T18:27:17,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437396) roll requested 2024-11-13T18:27:17,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437405 2024-11-13T18:27:17,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437405 2024-11-13T18:27:17,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437405) roll requested 2024-11-13T18:27:17,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437411 2024-11-13T18:27:17,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437411 2024-11-13T18:27:17,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437411) roll requested 2024-11-13T18:27:17,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437418 2024-11-13T18:27:17,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437418 2024-11-13T18:27:17,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437418) roll requested 2024-11-13T18:27:17,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437425 2024-11-13T18:27:17,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437425 2024-11-13T18:27:17,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437425) roll requested 2024-11-13T18:27:17,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437431 2024-11-13T18:27:17,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437431 2024-11-13T18:27:17,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437431) roll requested 2024-11-13T18:27:17,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437439 2024-11-13T18:27:17,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437439 2024-11-13T18:27:17,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437439) roll requested 2024-11-13T18:27:17,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437446 2024-11-13T18:27:17,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437446 2024-11-13T18:27:17,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437446) roll requested 2024-11-13T18:27:17,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437453 2024-11-13T18:27:17,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437453 2024-11-13T18:27:17,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437453) roll requested 2024-11-13T18:27:17,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437460 2024-11-13T18:27:17,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437460 2024-11-13T18:27:17,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437460) roll requested 2024-11-13T18:27:17,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437466 2024-11-13T18:27:17,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437466 2024-11-13T18:27:17,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437466) roll requested 2024-11-13T18:27:17,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437473 2024-11-13T18:27:17,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437473 2024-11-13T18:27:17,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437473) roll requested 2024-11-13T18:27:17,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437480 2024-11-13T18:27:17,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437480 2024-11-13T18:27:17,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437480) roll requested 2024-11-13T18:27:17,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437487 2024-11-13T18:27:17,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437487 2024-11-13T18:27:17,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437487) roll requested 2024-11-13T18:27:17,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437495 2024-11-13T18:27:17,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437495 2024-11-13T18:27:17,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437495) roll requested 2024-11-13T18:27:17,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437502 2024-11-13T18:27:17,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437502 2024-11-13T18:27:17,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437502) roll requested 2024-11-13T18:27:17,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437510 2024-11-13T18:27:17,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437510 2024-11-13T18:27:17,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437510) roll requested 2024-11-13T18:27:17,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437517 2024-11-13T18:27:17,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437517 2024-11-13T18:27:17,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437517) roll requested 2024-11-13T18:27:17,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437526 2024-11-13T18:27:17,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437526 2024-11-13T18:27:17,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437526) roll requested 2024-11-13T18:27:17,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437536 2024-11-13T18:27:17,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437536 2024-11-13T18:27:17,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437536) roll requested 2024-11-13T18:27:17,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437546 2024-11-13T18:27:17,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437546 2024-11-13T18:27:17,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437546) roll requested 2024-11-13T18:27:17,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437553 2024-11-13T18:27:17,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437553 2024-11-13T18:27:17,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437553) roll requested 2024-11-13T18:27:17,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437562 2024-11-13T18:27:17,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437562 2024-11-13T18:27:17,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437562) roll requested 2024-11-13T18:27:17,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437569 2024-11-13T18:27:17,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437569 2024-11-13T18:27:17,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437569) roll requested 2024-11-13T18:27:17,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437577 2024-11-13T18:27:17,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437577 2024-11-13T18:27:17,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437577) roll requested 2024-11-13T18:27:17,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437589 2024-11-13T18:27:17,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437589 2024-11-13T18:27:17,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437589) roll requested 2024-11-13T18:27:17,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437596 2024-11-13T18:27:17,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437596 2024-11-13T18:27:17,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437596) roll requested 2024-11-13T18:27:17,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437603 2024-11-13T18:27:17,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437603 2024-11-13T18:27:17,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437603) roll requested 2024-11-13T18:27:17,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437610 2024-11-13T18:27:17,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437610 2024-11-13T18:27:17,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437610) roll requested 2024-11-13T18:27:17,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437618 2024-11-13T18:27:17,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437618 2024-11-13T18:27:17,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437618) roll requested 2024-11-13T18:27:17,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437626 2024-11-13T18:27:17,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437626 2024-11-13T18:27:17,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437626) roll requested 2024-11-13T18:27:17,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437633 2024-11-13T18:27:17,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437633 2024-11-13T18:27:17,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437633) roll requested 2024-11-13T18:27:17,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437641 2024-11-13T18:27:17,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437641 2024-11-13T18:27:17,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437641) roll requested 2024-11-13T18:27:17,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437648 2024-11-13T18:27:17,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437648 2024-11-13T18:27:17,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437648) roll requested 2024-11-13T18:27:17,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437656 2024-11-13T18:27:17,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437656 2024-11-13T18:27:17,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437656) roll requested 2024-11-13T18:27:17,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437667 2024-11-13T18:27:17,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437667 2024-11-13T18:27:17,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437667) roll requested 2024-11-13T18:27:17,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437675 2024-11-13T18:27:17,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437675 2024-11-13T18:27:17,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437675) roll requested 2024-11-13T18:27:17,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437683 2024-11-13T18:27:17,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437683 2024-11-13T18:27:17,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437683) roll requested 2024-11-13T18:27:17,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437690 2024-11-13T18:27:17,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437690 2024-11-13T18:27:17,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437690) roll requested 2024-11-13T18:27:17,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437697 2024-11-13T18:27:17,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437697 2024-11-13T18:27:17,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437697) roll requested 2024-11-13T18:27:17,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437705 2024-11-13T18:27:17,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437705 2024-11-13T18:27:17,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437705) roll requested 2024-11-13T18:27:17,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437714 2024-11-13T18:27:17,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437714 2024-11-13T18:27:17,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437714) roll requested 2024-11-13T18:27:17,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437721 2024-11-13T18:27:17,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437721 2024-11-13T18:27:17,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437721) roll requested 2024-11-13T18:27:17,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437729 2024-11-13T18:27:17,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437729 2024-11-13T18:27:17,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437729) roll requested 2024-11-13T18:27:17,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437737 2024-11-13T18:27:17,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437737 2024-11-13T18:27:17,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437737) roll requested 2024-11-13T18:27:17,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437749 2024-11-13T18:27:17,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437749 2024-11-13T18:27:17,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437749) roll requested 2024-11-13T18:27:17,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437758 2024-11-13T18:27:17,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437758 2024-11-13T18:27:17,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437758) roll requested 2024-11-13T18:27:17,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437766 2024-11-13T18:27:17,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437766 2024-11-13T18:27:17,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437766) roll requested 2024-11-13T18:27:17,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437778 2024-11-13T18:27:17,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437778 2024-11-13T18:27:17,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437778) roll requested 2024-11-13T18:27:17,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437787 2024-11-13T18:27:17,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437787 2024-11-13T18:27:17,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437787) roll requested 2024-11-13T18:27:17,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437794 2024-11-13T18:27:17,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437794 2024-11-13T18:27:17,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437794) roll requested 2024-11-13T18:27:17,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437802 2024-11-13T18:27:17,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437802 2024-11-13T18:27:17,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437802) roll requested 2024-11-13T18:27:17,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437809 2024-11-13T18:27:17,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437809 2024-11-13T18:27:17,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437809) roll requested 2024-11-13T18:27:17,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437816 2024-11-13T18:27:17,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437816 2024-11-13T18:27:17,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437816) roll requested 2024-11-13T18:27:17,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437824 2024-11-13T18:27:17,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437824 2024-11-13T18:27:17,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437824) roll requested 2024-11-13T18:27:17,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437831 2024-11-13T18:27:17,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437831 2024-11-13T18:27:17,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437831) roll requested 2024-11-13T18:27:17,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437839 2024-11-13T18:27:17,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437839 2024-11-13T18:27:17,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437839) roll requested 2024-11-13T18:27:17,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437847 2024-11-13T18:27:17,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437847 2024-11-13T18:27:17,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437847) roll requested 2024-11-13T18:27:17,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437854 2024-11-13T18:27:17,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437854 2024-11-13T18:27:17,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437854) roll requested 2024-11-13T18:27:17,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437862 2024-11-13T18:27:17,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437862 2024-11-13T18:27:17,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437862) roll requested 2024-11-13T18:27:17,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437869 2024-11-13T18:27:17,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437869 2024-11-13T18:27:17,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437869) roll requested 2024-11-13T18:27:17,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437877 2024-11-13T18:27:17,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437877 2024-11-13T18:27:17,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437877) roll requested 2024-11-13T18:27:17,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437885 2024-11-13T18:27:17,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437885 2024-11-13T18:27:17,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437885) roll requested 2024-11-13T18:27:17,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437893 2024-11-13T18:27:17,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437893 2024-11-13T18:27:17,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437893) roll requested 2024-11-13T18:27:17,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437900 2024-11-13T18:27:17,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437900 2024-11-13T18:27:17,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437900) roll requested 2024-11-13T18:27:17,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437911 2024-11-13T18:27:17,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437911 2024-11-13T18:27:17,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437911) roll requested 2024-11-13T18:27:17,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437917 2024-11-13T18:27:17,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437917 2024-11-13T18:27:17,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437917) roll requested 2024-11-13T18:27:17,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437925 2024-11-13T18:27:17,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437925 2024-11-13T18:27:17,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437925) roll requested 2024-11-13T18:27:17,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437932 2024-11-13T18:27:17,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437932 2024-11-13T18:27:17,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437932) roll requested 2024-11-13T18:27:17,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437940 2024-11-13T18:27:17,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437940 2024-11-13T18:27:17,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437940) roll requested 2024-11-13T18:27:17,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437947 2024-11-13T18:27:17,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437947 2024-11-13T18:27:17,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437947) roll requested 2024-11-13T18:27:17,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437955 2024-11-13T18:27:17,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437955 2024-11-13T18:27:17,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437955) roll requested 2024-11-13T18:27:17,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437963 2024-11-13T18:27:17,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437963 2024-11-13T18:27:17,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437963) roll requested 2024-11-13T18:27:17,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437970 2024-11-13T18:27:17,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437970 2024-11-13T18:27:17,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437970) roll requested 2024-11-13T18:27:17,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437977 2024-11-13T18:27:17,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437977 2024-11-13T18:27:17,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437977) roll requested 2024-11-13T18:27:17,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437984 2024-11-13T18:27:17,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:17,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437984 2024-11-13T18:27:17,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:17,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:17,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437984) roll requested 2024-11-13T18:27:17,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522437997 2024-11-13T18:27:18,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437997 2024-11-13T18:27:18,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522437997) roll requested 2024-11-13T18:27:18,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438007 2024-11-13T18:27:18,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522437997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438007 2024-11-13T18:27:18,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438007) roll requested 2024-11-13T18:27:18,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438017 2024-11-13T18:27:18,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438017 2024-11-13T18:27:18,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438017) roll requested 2024-11-13T18:27:18,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438025 2024-11-13T18:27:18,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438025 2024-11-13T18:27:18,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438025) roll requested 2024-11-13T18:27:18,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438033 2024-11-13T18:27:18,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438033 2024-11-13T18:27:18,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438033) roll requested 2024-11-13T18:27:18,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438042 2024-11-13T18:27:18,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438042 2024-11-13T18:27:18,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438042) roll requested 2024-11-13T18:27:18,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438049 2024-11-13T18:27:18,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438049 2024-11-13T18:27:18,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438049) roll requested 2024-11-13T18:27:18,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438057 2024-11-13T18:27:18,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438057 2024-11-13T18:27:18,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438057) roll requested 2024-11-13T18:27:18,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438064 2024-11-13T18:27:18,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438064 2024-11-13T18:27:18,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438064) roll requested 2024-11-13T18:27:18,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438071 2024-11-13T18:27:18,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438071 2024-11-13T18:27:18,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438071) roll requested 2024-11-13T18:27:18,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438080 2024-11-13T18:27:18,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438080 2024-11-13T18:27:18,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438080) roll requested 2024-11-13T18:27:18,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438087 2024-11-13T18:27:18,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438087 2024-11-13T18:27:18,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438087) roll requested 2024-11-13T18:27:18,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438096 2024-11-13T18:27:18,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438096 2024-11-13T18:27:18,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438096) roll requested 2024-11-13T18:27:18,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438103 2024-11-13T18:27:18,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438103 2024-11-13T18:27:18,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438103) roll requested 2024-11-13T18:27:18,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438111 2024-11-13T18:27:18,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438111 2024-11-13T18:27:18,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438111) roll requested 2024-11-13T18:27:18,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438119 2024-11-13T18:27:18,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438119 2024-11-13T18:27:18,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438119) roll requested 2024-11-13T18:27:18,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438127 2024-11-13T18:27:18,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438127 2024-11-13T18:27:18,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438127) roll requested 2024-11-13T18:27:18,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438135 2024-11-13T18:27:18,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438135 2024-11-13T18:27:18,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438135) roll requested 2024-11-13T18:27:18,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438143 2024-11-13T18:27:18,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438143 2024-11-13T18:27:18,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438143) roll requested 2024-11-13T18:27:18,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438150 2024-11-13T18:27:18,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438150 2024-11-13T18:27:18,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438150) roll requested 2024-11-13T18:27:18,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438158 2024-11-13T18:27:18,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438158 2024-11-13T18:27:18,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438158) roll requested 2024-11-13T18:27:18,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438173 2024-11-13T18:27:18,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438173 2024-11-13T18:27:18,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438173) roll requested 2024-11-13T18:27:18,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438182 2024-11-13T18:27:18,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438182 2024-11-13T18:27:18,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438182) roll requested 2024-11-13T18:27:18,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438190 2024-11-13T18:27:18,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438190 2024-11-13T18:27:18,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438190) roll requested 2024-11-13T18:27:18,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438198 2024-11-13T18:27:18,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438198 2024-11-13T18:27:18,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438198) roll requested 2024-11-13T18:27:18,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438206 2024-11-13T18:27:18,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438206 2024-11-13T18:27:18,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438206) roll requested 2024-11-13T18:27:18,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438214 2024-11-13T18:27:18,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438214 2024-11-13T18:27:18,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438214) roll requested 2024-11-13T18:27:18,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438222 2024-11-13T18:27:18,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438222 2024-11-13T18:27:18,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438222) roll requested 2024-11-13T18:27:18,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438229 2024-11-13T18:27:18,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438229 2024-11-13T18:27:18,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438229) roll requested 2024-11-13T18:27:18,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438245 2024-11-13T18:27:18,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438245 2024-11-13T18:27:18,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438245) roll requested 2024-11-13T18:27:18,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438262 2024-11-13T18:27:18,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438262 2024-11-13T18:27:18,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438262) roll requested 2024-11-13T18:27:18,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438274 2024-11-13T18:27:18,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438274 2024-11-13T18:27:18,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438274) roll requested 2024-11-13T18:27:18,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438288 2024-11-13T18:27:18,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438288 2024-11-13T18:27:18,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438288) roll requested 2024-11-13T18:27:18,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438298 2024-11-13T18:27:18,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438298 2024-11-13T18:27:18,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438298) roll requested 2024-11-13T18:27:18,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438307 2024-11-13T18:27:18,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438307 2024-11-13T18:27:18,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438307) roll requested 2024-11-13T18:27:18,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438313 2024-11-13T18:27:18,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438313 2024-11-13T18:27:18,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438313) roll requested 2024-11-13T18:27:18,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438320 2024-11-13T18:27:18,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438320 2024-11-13T18:27:18,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438320) roll requested 2024-11-13T18:27:18,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438327 2024-11-13T18:27:18,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438327 2024-11-13T18:27:18,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438327) roll requested 2024-11-13T18:27:18,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438335 2024-11-13T18:27:18,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438335 2024-11-13T18:27:18,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438335) roll requested 2024-11-13T18:27:18,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438342 2024-11-13T18:27:18,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438342 2024-11-13T18:27:18,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438342) roll requested 2024-11-13T18:27:18,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438350 2024-11-13T18:27:18,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438350 2024-11-13T18:27:18,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438350) roll requested 2024-11-13T18:27:18,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438358 2024-11-13T18:27:18,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438358 2024-11-13T18:27:18,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438358) roll requested 2024-11-13T18:27:18,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438369 2024-11-13T18:27:18,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438369 2024-11-13T18:27:18,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438369) roll requested 2024-11-13T18:27:18,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438376 2024-11-13T18:27:18,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438376 2024-11-13T18:27:18,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438376) roll requested 2024-11-13T18:27:18,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438383 2024-11-13T18:27:18,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438383 2024-11-13T18:27:18,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438383) roll requested 2024-11-13T18:27:18,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438391 2024-11-13T18:27:18,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438391 2024-11-13T18:27:18,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438391) roll requested 2024-11-13T18:27:18,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438401 2024-11-13T18:27:18,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438401 2024-11-13T18:27:18,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438401) roll requested 2024-11-13T18:27:18,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438413 2024-11-13T18:27:18,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438413 2024-11-13T18:27:18,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438413) roll requested 2024-11-13T18:27:18,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438425 2024-11-13T18:27:18,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438425 2024-11-13T18:27:18,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438425) roll requested 2024-11-13T18:27:18,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438438 2024-11-13T18:27:18,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438438 2024-11-13T18:27:18,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438438) roll requested 2024-11-13T18:27:18,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438449 2024-11-13T18:27:18,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438449 2024-11-13T18:27:18,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438449) roll requested 2024-11-13T18:27:18,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438465 2024-11-13T18:27:18,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438465 2024-11-13T18:27:18,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438465) roll requested 2024-11-13T18:27:18,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438473 2024-11-13T18:27:18,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438473 2024-11-13T18:27:18,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438473) roll requested 2024-11-13T18:27:18,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438481 2024-11-13T18:27:18,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438481 2024-11-13T18:27:18,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438481) roll requested 2024-11-13T18:27:18,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438489 2024-11-13T18:27:18,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438489 2024-11-13T18:27:18,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438489) roll requested 2024-11-13T18:27:18,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438501 2024-11-13T18:27:18,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438501 2024-11-13T18:27:18,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438501) roll requested 2024-11-13T18:27:18,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438514 2024-11-13T18:27:18,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438514 2024-11-13T18:27:18,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438514) roll requested 2024-11-13T18:27:18,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438522 2024-11-13T18:27:18,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438522 2024-11-13T18:27:18,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438522) roll requested 2024-11-13T18:27:18,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438537 2024-11-13T18:27:18,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438537 2024-11-13T18:27:18,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438537) roll requested 2024-11-13T18:27:18,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438544 2024-11-13T18:27:18,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438544 2024-11-13T18:27:18,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438544) roll requested 2024-11-13T18:27:18,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438561 2024-11-13T18:27:18,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438561 2024-11-13T18:27:18,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438561) roll requested 2024-11-13T18:27:18,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438571 2024-11-13T18:27:18,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438571 2024-11-13T18:27:18,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438571) roll requested 2024-11-13T18:27:18,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438586 2024-11-13T18:27:18,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438586 2024-11-13T18:27:18,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438586) roll requested 2024-11-13T18:27:18,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438598 2024-11-13T18:27:18,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438598 2024-11-13T18:27:18,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438598) roll requested 2024-11-13T18:27:18,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438613 2024-11-13T18:27:18,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438613 2024-11-13T18:27:18,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438613) roll requested 2024-11-13T18:27:18,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438629 2024-11-13T18:27:18,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438629 2024-11-13T18:27:18,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438629) roll requested 2024-11-13T18:27:18,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438641 2024-11-13T18:27:18,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438641 2024-11-13T18:27:18,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438641) roll requested 2024-11-13T18:27:18,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438657 2024-11-13T18:27:18,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438657 2024-11-13T18:27:18,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438657) roll requested 2024-11-13T18:27:18,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438672 2024-11-13T18:27:18,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438672 2024-11-13T18:27:18,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438672) roll requested 2024-11-13T18:27:18,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438703 2024-11-13T18:27:18,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438703 2024-11-13T18:27:18,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438703) roll requested 2024-11-13T18:27:18,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438721 2024-11-13T18:27:18,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438721 2024-11-13T18:27:18,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438721) roll requested 2024-11-13T18:27:18,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438733 2024-11-13T18:27:18,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438733 2024-11-13T18:27:18,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438733) roll requested 2024-11-13T18:27:18,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438745 2024-11-13T18:27:18,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438745 2024-11-13T18:27:18,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438745) roll requested 2024-11-13T18:27:18,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438757 2024-11-13T18:27:18,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438757 2024-11-13T18:27:18,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438757) roll requested 2024-11-13T18:27:18,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438764 2024-11-13T18:27:18,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438764 2024-11-13T18:27:18,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438764) roll requested 2024-11-13T18:27:18,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438779 2024-11-13T18:27:18,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438779 2024-11-13T18:27:18,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438779) roll requested 2024-11-13T18:27:18,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438797 2024-11-13T18:27:18,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438797 2024-11-13T18:27:18,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438797) roll requested 2024-11-13T18:27:18,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438813 2024-11-13T18:27:18,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438813 2024-11-13T18:27:18,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438813) roll requested 2024-11-13T18:27:18,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438833 2024-11-13T18:27:18,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438833 2024-11-13T18:27:18,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438833) roll requested 2024-11-13T18:27:18,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438845 2024-11-13T18:27:18,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438845 2024-11-13T18:27:18,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438845) roll requested 2024-11-13T18:27:18,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438869 2024-11-13T18:27:18,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438869 2024-11-13T18:27:18,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438869) roll requested 2024-11-13T18:27:18,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438889 2024-11-13T18:27:18,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438889 2024-11-13T18:27:18,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438889) roll requested 2024-11-13T18:27:18,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438913 2024-11-13T18:27:18,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438913 2024-11-13T18:27:18,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438913) roll requested 2024-11-13T18:27:18,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438922 2024-11-13T18:27:18,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438922 2024-11-13T18:27:18,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438922) roll requested 2024-11-13T18:27:18,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438931 2024-11-13T18:27:18,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438931 2024-11-13T18:27:18,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438931) roll requested 2024-11-13T18:27:18,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438939 2024-11-13T18:27:18,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438939 2024-11-13T18:27:18,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438939) roll requested 2024-11-13T18:27:18,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438947 2024-11-13T18:27:18,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438947 2024-11-13T18:27:18,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438947) roll requested 2024-11-13T18:27:18,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438954 2024-11-13T18:27:18,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438954 2024-11-13T18:27:18,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438954) roll requested 2024-11-13T18:27:18,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438962 2024-11-13T18:27:18,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438962 2024-11-13T18:27:18,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438962) roll requested 2024-11-13T18:27:18,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438969 2024-11-13T18:27:18,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438969 2024-11-13T18:27:18,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438969) roll requested 2024-11-13T18:27:18,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438977 2024-11-13T18:27:18,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438977 2024-11-13T18:27:18,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:18,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438977) roll requested 2024-11-13T18:27:18,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522438984 2024-11-13T18:27:18,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:18,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438984 2024-11-13T18:27:18,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:18,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522438984) roll requested 2024-11-13T18:27:19,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439000 2024-11-13T18:27:19,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522438984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439000 2024-11-13T18:27:19,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439000) roll requested 2024-11-13T18:27:19,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439008 2024-11-13T18:27:19,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439008 2024-11-13T18:27:19,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439008) roll requested 2024-11-13T18:27:19,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439019 2024-11-13T18:27:19,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439019 2024-11-13T18:27:19,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439019) roll requested 2024-11-13T18:27:19,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439057 2024-11-13T18:27:19,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439057 2024-11-13T18:27:19,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439057) roll requested 2024-11-13T18:27:19,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439070 2024-11-13T18:27:19,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439070 2024-11-13T18:27:19,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439070) roll requested 2024-11-13T18:27:19,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439090 2024-11-13T18:27:19,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439090 2024-11-13T18:27:19,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439090) roll requested 2024-11-13T18:27:19,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439098 2024-11-13T18:27:19,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439098 2024-11-13T18:27:19,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439098) roll requested 2024-11-13T18:27:19,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439118 2024-11-13T18:27:19,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439118 2024-11-13T18:27:19,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439118) roll requested 2024-11-13T18:27:19,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439145 2024-11-13T18:27:19,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439145 2024-11-13T18:27:19,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439145) roll requested 2024-11-13T18:27:19,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439161 2024-11-13T18:27:19,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439161 2024-11-13T18:27:19,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439161) roll requested 2024-11-13T18:27:19,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439174 2024-11-13T18:27:19,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439174 2024-11-13T18:27:19,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439174) roll requested 2024-11-13T18:27:19,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439193 2024-11-13T18:27:19,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439193 2024-11-13T18:27:19,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439193) roll requested 2024-11-13T18:27:19,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439221 2024-11-13T18:27:19,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439221 2024-11-13T18:27:19,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439221) roll requested 2024-11-13T18:27:19,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439245 2024-11-13T18:27:19,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439245 2024-11-13T18:27:19,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439245) roll requested 2024-11-13T18:27:19,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439269 2024-11-13T18:27:19,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439269 2024-11-13T18:27:19,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439269) roll requested 2024-11-13T18:27:19,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439285 2024-11-13T18:27:19,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439285 2024-11-13T18:27:19,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439285) roll requested 2024-11-13T18:27:19,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439322 2024-11-13T18:27:19,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439322 2024-11-13T18:27:19,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439322) roll requested 2024-11-13T18:27:19,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439348 2024-11-13T18:27:19,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439348 2024-11-13T18:27:19,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439348) roll requested 2024-11-13T18:27:19,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439374 2024-11-13T18:27:19,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439374 2024-11-13T18:27:19,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439374) roll requested 2024-11-13T18:27:19,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439393 2024-11-13T18:27:19,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439393 2024-11-13T18:27:19,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439393) roll requested 2024-11-13T18:27:19,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439405 2024-11-13T18:27:19,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439405 2024-11-13T18:27:19,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439405) roll requested 2024-11-13T18:27:19,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439413 2024-11-13T18:27:19,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439413 2024-11-13T18:27:19,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439413) roll requested 2024-11-13T18:27:19,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439421 2024-11-13T18:27:19,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439421 2024-11-13T18:27:19,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439421) roll requested 2024-11-13T18:27:19,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439429 2024-11-13T18:27:19,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439429 2024-11-13T18:27:19,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439429) roll requested 2024-11-13T18:27:19,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439441 2024-11-13T18:27:19,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439441 2024-11-13T18:27:19,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439441) roll requested 2024-11-13T18:27:19,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439460 2024-11-13T18:27:19,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439460 2024-11-13T18:27:19,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439460) roll requested 2024-11-13T18:27:19,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439471 2024-11-13T18:27:19,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439471 2024-11-13T18:27:19,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439471) roll requested 2024-11-13T18:27:19,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439481 2024-11-13T18:27:19,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439481 2024-11-13T18:27:19,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439481) roll requested 2024-11-13T18:27:19,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439489 2024-11-13T18:27:19,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439489 2024-11-13T18:27:19,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439489) roll requested 2024-11-13T18:27:19,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439497 2024-11-13T18:27:19,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439497 2024-11-13T18:27:19,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439497) roll requested 2024-11-13T18:27:19,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439504 2024-11-13T18:27:19,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439504 2024-11-13T18:27:19,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439504) roll requested 2024-11-13T18:27:19,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439517 2024-11-13T18:27:19,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439517 2024-11-13T18:27:19,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439517) roll requested 2024-11-13T18:27:19,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439531 2024-11-13T18:27:19,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439531 2024-11-13T18:27:19,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439531) roll requested 2024-11-13T18:27:19,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439541 2024-11-13T18:27:19,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439541 2024-11-13T18:27:19,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439541) roll requested 2024-11-13T18:27:19,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439555 2024-11-13T18:27:19,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439555 2024-11-13T18:27:19,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439555) roll requested 2024-11-13T18:27:19,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439567 2024-11-13T18:27:19,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439567 2024-11-13T18:27:19,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439567) roll requested 2024-11-13T18:27:19,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439584 2024-11-13T18:27:19,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439584 2024-11-13T18:27:19,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439584) roll requested 2024-11-13T18:27:19,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439601 2024-11-13T18:27:19,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439601 2024-11-13T18:27:19,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439601) roll requested 2024-11-13T18:27:19,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439625 2024-11-13T18:27:19,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439625 2024-11-13T18:27:19,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439625) roll requested 2024-11-13T18:27:19,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439648 2024-11-13T18:27:19,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439648 2024-11-13T18:27:19,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439648) roll requested 2024-11-13T18:27:19,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439681 2024-11-13T18:27:19,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439681 2024-11-13T18:27:19,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439681) roll requested 2024-11-13T18:27:19,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439694 2024-11-13T18:27:19,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439694 2024-11-13T18:27:19,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439694) roll requested 2024-11-13T18:27:19,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439709 2024-11-13T18:27:19,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439709 2024-11-13T18:27:19,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439709) roll requested 2024-11-13T18:27:19,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439722 2024-11-13T18:27:19,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439722 2024-11-13T18:27:19,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439722) roll requested 2024-11-13T18:27:19,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439730 2024-11-13T18:27:19,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439730 2024-11-13T18:27:19,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439730) roll requested 2024-11-13T18:27:19,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439738 2024-11-13T18:27:19,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439738 2024-11-13T18:27:19,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439738) roll requested 2024-11-13T18:27:19,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439745 2024-11-13T18:27:19,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439745 2024-11-13T18:27:19,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439745) roll requested 2024-11-13T18:27:19,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439752 2024-11-13T18:27:19,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439752 2024-11-13T18:27:19,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439752) roll requested 2024-11-13T18:27:19,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439760 2024-11-13T18:27:19,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439760 2024-11-13T18:27:19,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439760) roll requested 2024-11-13T18:27:19,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439773 2024-11-13T18:27:19,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439773 2024-11-13T18:27:19,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439773) roll requested 2024-11-13T18:27:19,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439791 2024-11-13T18:27:19,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439791 2024-11-13T18:27:19,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439791) roll requested 2024-11-13T18:27:19,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439817 2024-11-13T18:27:19,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439817 2024-11-13T18:27:19,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439817) roll requested 2024-11-13T18:27:19,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439833 2024-11-13T18:27:19,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439833 2024-11-13T18:27:19,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439833) roll requested 2024-11-13T18:27:19,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439845 2024-11-13T18:27:19,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439845 2024-11-13T18:27:19,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439845) roll requested 2024-11-13T18:27:19,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439857 2024-11-13T18:27:19,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439857 2024-11-13T18:27:19,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439857) roll requested 2024-11-13T18:27:19,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439865 2024-11-13T18:27:19,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439865 2024-11-13T18:27:19,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439865) roll requested 2024-11-13T18:27:19,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439873 2024-11-13T18:27:19,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439873 2024-11-13T18:27:19,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439873) roll requested 2024-11-13T18:27:19,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439880 2024-11-13T18:27:19,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439880 2024-11-13T18:27:19,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439880) roll requested 2024-11-13T18:27:19,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439887 2024-11-13T18:27:19,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439887 2024-11-13T18:27:19,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439887) roll requested 2024-11-13T18:27:19,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439895 2024-11-13T18:27:19,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439895 2024-11-13T18:27:19,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439895) roll requested 2024-11-13T18:27:19,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439904 2024-11-13T18:27:19,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439904 2024-11-13T18:27:19,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439904) roll requested 2024-11-13T18:27:19,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439912 2024-11-13T18:27:19,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439912 2024-11-13T18:27:19,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439912) roll requested 2024-11-13T18:27:19,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439921 2024-11-13T18:27:19,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439921 2024-11-13T18:27:19,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439921) roll requested 2024-11-13T18:27:19,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439935 2024-11-13T18:27:19,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439935 2024-11-13T18:27:19,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439935) roll requested 2024-11-13T18:27:19,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439942 2024-11-13T18:27:19,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439942 2024-11-13T18:27:19,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439942) roll requested 2024-11-13T18:27:19,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439949 2024-11-13T18:27:19,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439949 2024-11-13T18:27:19,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439949) roll requested 2024-11-13T18:27:19,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439961 2024-11-13T18:27:19,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439961 2024-11-13T18:27:19,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439961) roll requested 2024-11-13T18:27:19,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439968 2024-11-13T18:27:19,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439968 2024-11-13T18:27:19,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439968) roll requested 2024-11-13T18:27:19,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439977 2024-11-13T18:27:19,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439977 2024-11-13T18:27:19,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439977) roll requested 2024-11-13T18:27:19,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439985 2024-11-13T18:27:19,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439985 2024-11-13T18:27:19,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:19,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:19,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439985) roll requested 2024-11-13T18:27:19,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522439992 2024-11-13T18:27:19,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:19,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439992 2024-11-13T18:27:20,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522439992) roll requested 2024-11-13T18:27:20,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440001 2024-11-13T18:27:20,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522439992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440001 2024-11-13T18:27:20,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440001) roll requested 2024-11-13T18:27:20,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440009 2024-11-13T18:27:20,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440009 2024-11-13T18:27:20,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440009) roll requested 2024-11-13T18:27:20,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440020 2024-11-13T18:27:20,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440020 2024-11-13T18:27:20,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440020) roll requested 2024-11-13T18:27:20,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440029 2024-11-13T18:27:20,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440029 2024-11-13T18:27:20,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440029) roll requested 2024-11-13T18:27:20,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440041 2024-11-13T18:27:20,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440041 2024-11-13T18:27:20,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440041) roll requested 2024-11-13T18:27:20,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440054 2024-11-13T18:27:20,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440054 2024-11-13T18:27:20,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440054) roll requested 2024-11-13T18:27:20,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440073 2024-11-13T18:27:20,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440073 2024-11-13T18:27:20,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440073) roll requested 2024-11-13T18:27:20,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440082 2024-11-13T18:27:20,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440082 2024-11-13T18:27:20,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440082) roll requested 2024-11-13T18:27:20,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440091 2024-11-13T18:27:20,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440091 2024-11-13T18:27:20,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440091) roll requested 2024-11-13T18:27:20,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440098 2024-11-13T18:27:20,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440098 2024-11-13T18:27:20,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440098) roll requested 2024-11-13T18:27:20,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440105 2024-11-13T18:27:20,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440105 2024-11-13T18:27:20,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440105) roll requested 2024-11-13T18:27:20,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440112 2024-11-13T18:27:20,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440112 2024-11-13T18:27:20,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440112) roll requested 2024-11-13T18:27:20,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440121 2024-11-13T18:27:20,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440121 2024-11-13T18:27:20,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440121) roll requested 2024-11-13T18:27:20,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440128 2024-11-13T18:27:20,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440128 2024-11-13T18:27:20,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440128) roll requested 2024-11-13T18:27:20,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440136 2024-11-13T18:27:20,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440136 2024-11-13T18:27:20,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440136) roll requested 2024-11-13T18:27:20,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440143 2024-11-13T18:27:20,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440143 2024-11-13T18:27:20,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440143) roll requested 2024-11-13T18:27:20,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440150 2024-11-13T18:27:20,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440150 2024-11-13T18:27:20,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440150) roll requested 2024-11-13T18:27:20,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440157 2024-11-13T18:27:20,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440157 2024-11-13T18:27:20,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440157) roll requested 2024-11-13T18:27:20,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440163 2024-11-13T18:27:20,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440163 2024-11-13T18:27:20,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440163) roll requested 2024-11-13T18:27:20,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440171 2024-11-13T18:27:20,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440171 2024-11-13T18:27:20,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440171) roll requested 2024-11-13T18:27:20,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440178 2024-11-13T18:27:20,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440178 2024-11-13T18:27:20,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440178) roll requested 2024-11-13T18:27:20,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440186 2024-11-13T18:27:20,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440186 2024-11-13T18:27:20,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440186) roll requested 2024-11-13T18:27:20,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440193 2024-11-13T18:27:20,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440193 2024-11-13T18:27:20,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440193) roll requested 2024-11-13T18:27:20,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440201 2024-11-13T18:27:20,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440201 2024-11-13T18:27:20,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440201) roll requested 2024-11-13T18:27:20,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440209 2024-11-13T18:27:20,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440209 2024-11-13T18:27:20,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440209) roll requested 2024-11-13T18:27:20,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440220 2024-11-13T18:27:20,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440220 2024-11-13T18:27:20,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440220) roll requested 2024-11-13T18:27:20,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440227 2024-11-13T18:27:20,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440227 2024-11-13T18:27:20,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440227) roll requested 2024-11-13T18:27:20,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440238 2024-11-13T18:27:20,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440238 2024-11-13T18:27:20,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440238) roll requested 2024-11-13T18:27:20,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440246 2024-11-13T18:27:20,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440246 2024-11-13T18:27:20,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440246) roll requested 2024-11-13T18:27:20,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440257 2024-11-13T18:27:20,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440257 2024-11-13T18:27:20,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440257) roll requested 2024-11-13T18:27:20,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440265 2024-11-13T18:27:20,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440265 2024-11-13T18:27:20,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440265) roll requested 2024-11-13T18:27:20,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440274 2024-11-13T18:27:20,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440274 2024-11-13T18:27:20,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440274) roll requested 2024-11-13T18:27:20,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440285 2024-11-13T18:27:20,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440285 2024-11-13T18:27:20,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440285) roll requested 2024-11-13T18:27:20,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440293 2024-11-13T18:27:20,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440293 2024-11-13T18:27:20,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440293) roll requested 2024-11-13T18:27:20,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440300 2024-11-13T18:27:20,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440300 2024-11-13T18:27:20,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440300) roll requested 2024-11-13T18:27:20,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440308 2024-11-13T18:27:20,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440308 2024-11-13T18:27:20,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440308) roll requested 2024-11-13T18:27:20,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440321 2024-11-13T18:27:20,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440321 2024-11-13T18:27:20,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440321) roll requested 2024-11-13T18:27:20,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440336 2024-11-13T18:27:20,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440336 2024-11-13T18:27:20,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440336) roll requested 2024-11-13T18:27:20,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440344 2024-11-13T18:27:20,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440344 2024-11-13T18:27:20,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440344) roll requested 2024-11-13T18:27:20,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440352 2024-11-13T18:27:20,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440352 2024-11-13T18:27:20,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440352) roll requested 2024-11-13T18:27:20,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440359 2024-11-13T18:27:20,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440359 2024-11-13T18:27:20,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440359) roll requested 2024-11-13T18:27:20,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440365 2024-11-13T18:27:20,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440365 2024-11-13T18:27:20,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440365) roll requested 2024-11-13T18:27:20,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440372 2024-11-13T18:27:20,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440372 2024-11-13T18:27:20,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440372) roll requested 2024-11-13T18:27:20,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440379 2024-11-13T18:27:20,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440379 2024-11-13T18:27:20,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440379) roll requested 2024-11-13T18:27:20,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440386 2024-11-13T18:27:20,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440386 2024-11-13T18:27:20,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440386) roll requested 2024-11-13T18:27:20,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440394 2024-11-13T18:27:20,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440394 2024-11-13T18:27:20,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440394) roll requested 2024-11-13T18:27:20,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440400 2024-11-13T18:27:20,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440400 2024-11-13T18:27:20,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440400) roll requested 2024-11-13T18:27:20,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440407 2024-11-13T18:27:20,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440407 2024-11-13T18:27:20,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440407) roll requested 2024-11-13T18:27:20,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440415 2024-11-13T18:27:20,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440415 2024-11-13T18:27:20,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440415) roll requested 2024-11-13T18:27:20,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440422 2024-11-13T18:27:20,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440422 2024-11-13T18:27:20,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440422) roll requested 2024-11-13T18:27:20,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440430 2024-11-13T18:27:20,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440430 2024-11-13T18:27:20,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440430) roll requested 2024-11-13T18:27:20,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440440 2024-11-13T18:27:20,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440440 2024-11-13T18:27:20,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440440) roll requested 2024-11-13T18:27:20,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440448 2024-11-13T18:27:20,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440448 2024-11-13T18:27:20,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440448) roll requested 2024-11-13T18:27:20,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440456 2024-11-13T18:27:20,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440456 2024-11-13T18:27:20,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440456) roll requested 2024-11-13T18:27:20,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440463 2024-11-13T18:27:20,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440463 2024-11-13T18:27:20,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440463) roll requested 2024-11-13T18:27:20,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440471 2024-11-13T18:27:20,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440471 2024-11-13T18:27:20,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440471) roll requested 2024-11-13T18:27:20,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440479 2024-11-13T18:27:20,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440479 2024-11-13T18:27:20,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440479) roll requested 2024-11-13T18:27:20,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440486 2024-11-13T18:27:20,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440486 2024-11-13T18:27:20,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440486) roll requested 2024-11-13T18:27:20,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440493 2024-11-13T18:27:20,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440493 2024-11-13T18:27:20,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440493) roll requested 2024-11-13T18:27:20,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440500 2024-11-13T18:27:20,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440500 2024-11-13T18:27:20,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440500) roll requested 2024-11-13T18:27:20,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440508 2024-11-13T18:27:20,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440508 2024-11-13T18:27:20,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440508) roll requested 2024-11-13T18:27:20,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440515 2024-11-13T18:27:20,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440515 2024-11-13T18:27:20,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440515) roll requested 2024-11-13T18:27:20,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440523 2024-11-13T18:27:20,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440523 2024-11-13T18:27:20,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440523) roll requested 2024-11-13T18:27:20,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440531 2024-11-13T18:27:20,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440531 2024-11-13T18:27:20,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440531) roll requested 2024-11-13T18:27:20,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440541 2024-11-13T18:27:20,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440541 2024-11-13T18:27:20,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440541) roll requested 2024-11-13T18:27:20,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440548 2024-11-13T18:27:20,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440548 2024-11-13T18:27:20,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440548) roll requested 2024-11-13T18:27:20,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440556 2024-11-13T18:27:20,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440556 2024-11-13T18:27:20,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440556) roll requested 2024-11-13T18:27:20,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440569 2024-11-13T18:27:20,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440569 2024-11-13T18:27:20,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440569) roll requested 2024-11-13T18:27:20,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440576 2024-11-13T18:27:20,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440576 2024-11-13T18:27:20,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440576) roll requested 2024-11-13T18:27:20,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440589 2024-11-13T18:27:20,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440589 2024-11-13T18:27:20,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440589) roll requested 2024-11-13T18:27:20,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440598 2024-11-13T18:27:20,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440598 2024-11-13T18:27:20,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440598) roll requested 2024-11-13T18:27:20,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440605 2024-11-13T18:27:20,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440605 2024-11-13T18:27:20,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440605) roll requested 2024-11-13T18:27:20,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440612 2024-11-13T18:27:20,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440612 2024-11-13T18:27:20,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440612) roll requested 2024-11-13T18:27:20,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440625 2024-11-13T18:27:20,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440625 2024-11-13T18:27:20,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440625) roll requested 2024-11-13T18:27:20,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440632 2024-11-13T18:27:20,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440632 2024-11-13T18:27:20,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440632) roll requested 2024-11-13T18:27:20,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440639 2024-11-13T18:27:20,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440639 2024-11-13T18:27:20,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440639) roll requested 2024-11-13T18:27:20,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440646 2024-11-13T18:27:20,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440646 2024-11-13T18:27:20,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440646) roll requested 2024-11-13T18:27:20,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440654 2024-11-13T18:27:20,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440654 2024-11-13T18:27:20,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440654) roll requested 2024-11-13T18:27:20,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440660 2024-11-13T18:27:20,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440660 2024-11-13T18:27:20,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440660) roll requested 2024-11-13T18:27:20,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440667 2024-11-13T18:27:20,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440667 2024-11-13T18:27:20,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440667) roll requested 2024-11-13T18:27:20,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440678 2024-11-13T18:27:20,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440678 2024-11-13T18:27:20,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440678) roll requested 2024-11-13T18:27:20,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440685 2024-11-13T18:27:20,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440685 2024-11-13T18:27:20,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440685) roll requested 2024-11-13T18:27:20,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440693 2024-11-13T18:27:20,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440693 2024-11-13T18:27:20,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440693) roll requested 2024-11-13T18:27:20,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440700 2024-11-13T18:27:20,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440700 2024-11-13T18:27:20,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440700) roll requested 2024-11-13T18:27:20,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440708 2024-11-13T18:27:20,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440708 2024-11-13T18:27:20,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440708) roll requested 2024-11-13T18:27:20,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440714 2024-11-13T18:27:20,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440714 2024-11-13T18:27:20,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440714) roll requested 2024-11-13T18:27:20,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440722 2024-11-13T18:27:20,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440722 2024-11-13T18:27:20,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440722) roll requested 2024-11-13T18:27:20,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440729 2024-11-13T18:27:20,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440729 2024-11-13T18:27:20,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440729) roll requested 2024-11-13T18:27:20,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440736 2024-11-13T18:27:20,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440736 2024-11-13T18:27:20,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440736) roll requested 2024-11-13T18:27:20,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440743 2024-11-13T18:27:20,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440743 2024-11-13T18:27:20,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440743) roll requested 2024-11-13T18:27:20,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440751 2024-11-13T18:27:20,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440751 2024-11-13T18:27:20,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440751) roll requested 2024-11-13T18:27:20,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440758 2024-11-13T18:27:20,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440758 2024-11-13T18:27:20,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440758) roll requested 2024-11-13T18:27:20,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440765 2024-11-13T18:27:20,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440765 2024-11-13T18:27:20,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440765) roll requested 2024-11-13T18:27:20,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440773 2024-11-13T18:27:20,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440773 2024-11-13T18:27:20,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440773) roll requested 2024-11-13T18:27:20,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440785 2024-11-13T18:27:20,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440785 2024-11-13T18:27:20,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440785) roll requested 2024-11-13T18:27:20,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440793 2024-11-13T18:27:20,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440793 2024-11-13T18:27:20,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440793) roll requested 2024-11-13T18:27:20,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440800 2024-11-13T18:27:20,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440800 2024-11-13T18:27:20,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440800) roll requested 2024-11-13T18:27:20,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440808 2024-11-13T18:27:20,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440808 2024-11-13T18:27:20,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440808) roll requested 2024-11-13T18:27:20,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440819 2024-11-13T18:27:20,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440819 2024-11-13T18:27:20,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440819) roll requested 2024-11-13T18:27:20,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440833 2024-11-13T18:27:20,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440833 2024-11-13T18:27:20,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440833) roll requested 2024-11-13T18:27:20,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440841 2024-11-13T18:27:20,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440841 2024-11-13T18:27:20,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440841) roll requested 2024-11-13T18:27:20,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440849 2024-11-13T18:27:20,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440849 2024-11-13T18:27:20,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440849) roll requested 2024-11-13T18:27:20,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440857 2024-11-13T18:27:20,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440857 2024-11-13T18:27:20,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440857) roll requested 2024-11-13T18:27:20,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440869 2024-11-13T18:27:20,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440869 2024-11-13T18:27:20,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440869) roll requested 2024-11-13T18:27:20,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440881 2024-11-13T18:27:20,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440881 2024-11-13T18:27:20,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440881) roll requested 2024-11-13T18:27:20,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440891 2024-11-13T18:27:20,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440891 2024-11-13T18:27:20,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440891) roll requested 2024-11-13T18:27:20,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440899 2024-11-13T18:27:20,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440899 2024-11-13T18:27:20,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440899) roll requested 2024-11-13T18:27:20,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440907 2024-11-13T18:27:20,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440907 2024-11-13T18:27:20,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440907) roll requested 2024-11-13T18:27:20,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440914 2024-11-13T18:27:20,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440914 2024-11-13T18:27:20,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440914) roll requested 2024-11-13T18:27:20,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440920 2024-11-13T18:27:20,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440920 2024-11-13T18:27:20,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440920) roll requested 2024-11-13T18:27:20,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440927 2024-11-13T18:27:20,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440927 2024-11-13T18:27:20,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440927) roll requested 2024-11-13T18:27:20,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440934 2024-11-13T18:27:20,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440934 2024-11-13T18:27:20,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440934) roll requested 2024-11-13T18:27:20,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440942 2024-11-13T18:27:20,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440942 2024-11-13T18:27:20,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440942) roll requested 2024-11-13T18:27:20,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440949 2024-11-13T18:27:20,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440949 2024-11-13T18:27:20,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440949) roll requested 2024-11-13T18:27:20,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440957 2024-11-13T18:27:20,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440957 2024-11-13T18:27:20,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440957) roll requested 2024-11-13T18:27:20,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440964 2024-11-13T18:27:20,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440964 2024-11-13T18:27:20,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440964) roll requested 2024-11-13T18:27:20,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440972 2024-11-13T18:27:20,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440972 2024-11-13T18:27:20,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440972) roll requested 2024-11-13T18:27:20,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440980 2024-11-13T18:27:20,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:20,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440980 2024-11-13T18:27:20,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:20,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:20,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440980) roll requested 2024-11-13T18:27:20,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522440994 2024-11-13T18:27:21,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440994 2024-11-13T18:27:21,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522440994) roll requested 2024-11-13T18:27:21,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441009 2024-11-13T18:27:21,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441009 2024-11-13T18:27:21,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522440994 is not closed yet, will try archiving it next time 2024-11-13T18:27:21,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441009) roll requested 2024-11-13T18:27:21,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441017 2024-11-13T18:27:21,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441017 2024-11-13T18:27:21,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441017) roll requested 2024-11-13T18:27:21,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441029 2024-11-13T18:27:21,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441029 2024-11-13T18:27:21,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441029) roll requested 2024-11-13T18:27:21,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441037 2024-11-13T18:27:21,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441037 2024-11-13T18:27:21,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441037) roll requested 2024-11-13T18:27:21,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441047 2024-11-13T18:27:21,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441047 2024-11-13T18:27:21,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441047) roll requested 2024-11-13T18:27:21,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441067 2024-11-13T18:27:21,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441067 2024-11-13T18:27:21,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441067) roll requested 2024-11-13T18:27:21,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441078 2024-11-13T18:27:21,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441078 2024-11-13T18:27:21,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441078) roll requested 2024-11-13T18:27:21,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441087 2024-11-13T18:27:21,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441087 2024-11-13T18:27:21,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441087) roll requested 2024-11-13T18:27:21,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441097 2024-11-13T18:27:21,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441097 2024-11-13T18:27:21,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441087 is not closed yet, will try archiving it next time 2024-11-13T18:27:21,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441097) roll requested 2024-11-13T18:27:21,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441106 2024-11-13T18:27:21,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441106 2024-11-13T18:27:21,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441106) roll requested 2024-11-13T18:27:21,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441114 2024-11-13T18:27:21,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441114 2024-11-13T18:27:21,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441114) roll requested 2024-11-13T18:27:21,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441127 2024-11-13T18:27:21,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441127 2024-11-13T18:27:21,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441127) roll requested 2024-11-13T18:27:21,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441141 2024-11-13T18:27:21,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441141 2024-11-13T18:27:21,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441141) roll requested 2024-11-13T18:27:21,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441161 2024-11-13T18:27:21,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441161 2024-11-13T18:27:21,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441161) roll requested 2024-11-13T18:27:21,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441205 2024-11-13T18:27:21,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441205 2024-11-13T18:27:21,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441161 is not closed yet, will try archiving it next time 2024-11-13T18:27:21,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441205) roll requested 2024-11-13T18:27:21,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441216 2024-11-13T18:27:21,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441216 2024-11-13T18:27:21,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441216) roll requested 2024-11-13T18:27:21,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441232 2024-11-13T18:27:21,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441232 2024-11-13T18:27:21,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441232) roll requested 2024-11-13T18:27:21,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441243 2024-11-13T18:27:21,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441243 2024-11-13T18:27:21,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441243) roll requested 2024-11-13T18:27:21,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441257 2024-11-13T18:27:21,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441257 2024-11-13T18:27:21,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441257) roll requested 2024-11-13T18:27:21,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441265 2024-11-13T18:27:21,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441265 2024-11-13T18:27:21,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441265) roll requested 2024-11-13T18:27:21,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441274 2024-11-13T18:27:21,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441274 2024-11-13T18:27:21,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441274) roll requested 2024-11-13T18:27:21,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441282 2024-11-13T18:27:21,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441282 2024-11-13T18:27:21,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441282) roll requested 2024-11-13T18:27:21,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441290 2024-11-13T18:27:21,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441290 2024-11-13T18:27:21,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441290) roll requested 2024-11-13T18:27:21,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441299 2024-11-13T18:27:21,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441299 2024-11-13T18:27:21,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441299) roll requested 2024-11-13T18:27:21,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441321 2024-11-13T18:27:21,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441321 2024-11-13T18:27:21,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441321) roll requested 2024-11-13T18:27:21,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441330 2024-11-13T18:27:21,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441330 2024-11-13T18:27:21,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441330) roll requested 2024-11-13T18:27:21,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441339 2024-11-13T18:27:21,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441339 2024-11-13T18:27:21,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441339) roll requested 2024-11-13T18:27:21,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441354 2024-11-13T18:27:21,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441354 2024-11-13T18:27:21,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441354) roll requested 2024-11-13T18:27:21,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441364 2024-11-13T18:27:21,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441364 2024-11-13T18:27:21,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441364) roll requested 2024-11-13T18:27:21,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441380 2024-11-13T18:27:21,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441380 2024-11-13T18:27:21,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441380) roll requested 2024-11-13T18:27:21,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441397 2024-11-13T18:27:21,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441397 2024-11-13T18:27:21,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441397) roll requested 2024-11-13T18:27:21,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441406 2024-11-13T18:27:21,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441406 2024-11-13T18:27:21,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441406) roll requested 2024-11-13T18:27:21,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441433 2024-11-13T18:27:21,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441433 2024-11-13T18:27:21,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441433) roll requested 2024-11-13T18:27:21,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441465 2024-11-13T18:27:21,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441465 2024-11-13T18:27:21,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441465) roll requested 2024-11-13T18:27:21,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441477 2024-11-13T18:27:21,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441477 2024-11-13T18:27:21,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441477) roll requested 2024-11-13T18:27:21,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441490 2024-11-13T18:27:21,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441490 2024-11-13T18:27:21,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441490) roll requested 2024-11-13T18:27:21,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441501 2024-11-13T18:27:21,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441501 2024-11-13T18:27:21,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441501) roll requested 2024-11-13T18:27:21,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441517 2024-11-13T18:27:21,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441517 2024-11-13T18:27:21,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441517) roll requested 2024-11-13T18:27:21,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441537 2024-11-13T18:27:21,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441537 2024-11-13T18:27:21,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441537) roll requested 2024-11-13T18:27:21,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441546 2024-11-13T18:27:21,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441546 2024-11-13T18:27:21,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441546) roll requested 2024-11-13T18:27:21,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441554 2024-11-13T18:27:21,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441554 2024-11-13T18:27:21,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441554) roll requested 2024-11-13T18:27:21,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441561 2024-11-13T18:27:21,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441561 2024-11-13T18:27:21,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441561) roll requested 2024-11-13T18:27:21,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441568 2024-11-13T18:27:21,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441568 2024-11-13T18:27:21,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441568) roll requested 2024-11-13T18:27:21,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441575 2024-11-13T18:27:21,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441575 2024-11-13T18:27:21,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441575) roll requested 2024-11-13T18:27:21,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441583 2024-11-13T18:27:21,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441583 2024-11-13T18:27:21,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441583) roll requested 2024-11-13T18:27:21,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441595 2024-11-13T18:27:21,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441595 2024-11-13T18:27:21,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441595) roll requested 2024-11-13T18:27:21,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441603 2024-11-13T18:27:21,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441603 2024-11-13T18:27:21,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441603) roll requested 2024-11-13T18:27:21,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441617 2024-11-13T18:27:21,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441617 2024-11-13T18:27:21,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441617) roll requested 2024-11-13T18:27:21,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441629 2024-11-13T18:27:21,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441629 2024-11-13T18:27:21,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441629) roll requested 2024-11-13T18:27:21,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441637 2024-11-13T18:27:21,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441637 2024-11-13T18:27:21,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441637) roll requested 2024-11-13T18:27:21,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441645 2024-11-13T18:27:21,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441645 2024-11-13T18:27:21,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441645) roll requested 2024-11-13T18:27:21,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441653 2024-11-13T18:27:21,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441653 2024-11-13T18:27:21,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441653) roll requested 2024-11-13T18:27:21,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441660 2024-11-13T18:27:21,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441660 2024-11-13T18:27:21,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441660) roll requested 2024-11-13T18:27:21,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441668 2024-11-13T18:27:21,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441668 2024-11-13T18:27:21,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441668) roll requested 2024-11-13T18:27:21,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441680 2024-11-13T18:27:21,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441680 2024-11-13T18:27:21,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441680) roll requested 2024-11-13T18:27:21,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441689 2024-11-13T18:27:21,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441689 2024-11-13T18:27:21,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441689) roll requested 2024-11-13T18:27:21,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441696 2024-11-13T18:27:21,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441696 2024-11-13T18:27:21,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441696) roll requested 2024-11-13T18:27:21,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441704 2024-11-13T18:27:21,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441704 2024-11-13T18:27:21,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441704) roll requested 2024-11-13T18:27:21,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441717 2024-11-13T18:27:21,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441717 2024-11-13T18:27:21,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441717) roll requested 2024-11-13T18:27:21,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441724 2024-11-13T18:27:21,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441724 2024-11-13T18:27:21,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441724) roll requested 2024-11-13T18:27:21,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441737 2024-11-13T18:27:21,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441737 2024-11-13T18:27:21,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441737) roll requested 2024-11-13T18:27:21,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441749 2024-11-13T18:27:21,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441749 2024-11-13T18:27:21,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441749) roll requested 2024-11-13T18:27:21,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441759 2024-11-13T18:27:21,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441759 2024-11-13T18:27:21,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441759) roll requested 2024-11-13T18:27:21,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441773 2024-11-13T18:27:21,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441773 2024-11-13T18:27:21,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441773) roll requested 2024-11-13T18:27:21,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441786 2024-11-13T18:27:21,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441786 2024-11-13T18:27:21,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441786) roll requested 2024-11-13T18:27:21,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441794 2024-11-13T18:27:21,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441794 2024-11-13T18:27:21,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441794) roll requested 2024-11-13T18:27:21,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441808 2024-11-13T18:27:21,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441808 2024-11-13T18:27:21,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441808) roll requested 2024-11-13T18:27:21,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441817 2024-11-13T18:27:21,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441817 2024-11-13T18:27:21,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441817) roll requested 2024-11-13T18:27:21,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441829 2024-11-13T18:27:21,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441829 2024-11-13T18:27:21,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441829) roll requested 2024-11-13T18:27:21,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441837 2024-11-13T18:27:21,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441837 2024-11-13T18:27:21,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441837) roll requested 2024-11-13T18:27:21,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441845 2024-11-13T18:27:21,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441845 2024-11-13T18:27:21,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441845) roll requested 2024-11-13T18:27:21,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441856 2024-11-13T18:27:21,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441856 2024-11-13T18:27:21,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441856) roll requested 2024-11-13T18:27:21,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441864 2024-11-13T18:27:21,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441864 2024-11-13T18:27:21,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441864) roll requested 2024-11-13T18:27:21,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441878 2024-11-13T18:27:21,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441878 2024-11-13T18:27:21,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441878) roll requested 2024-11-13T18:27:21,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441888 2024-11-13T18:27:21,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441888 2024-11-13T18:27:21,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441888) roll requested 2024-11-13T18:27:21,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441916 2024-11-13T18:27:21,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441916 2024-11-13T18:27:21,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441916) roll requested 2024-11-13T18:27:21,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441924 2024-11-13T18:27:21,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441924 2024-11-13T18:27:21,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441924) roll requested 2024-11-13T18:27:21,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441931 2024-11-13T18:27:21,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441931 2024-11-13T18:27:21,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441931) roll requested 2024-11-13T18:27:21,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441938 2024-11-13T18:27:21,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441938 2024-11-13T18:27:21,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441938) roll requested 2024-11-13T18:27:21,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441945 2024-11-13T18:27:21,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441945 2024-11-13T18:27:21,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441945) roll requested 2024-11-13T18:27:21,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441952 2024-11-13T18:27:21,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441952 2024-11-13T18:27:21,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441952) roll requested 2024-11-13T18:27:21,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441959 2024-11-13T18:27:21,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441959 2024-11-13T18:27:21,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441959) roll requested 2024-11-13T18:27:21,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441967 2024-11-13T18:27:21,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441967 2024-11-13T18:27:21,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441967) roll requested 2024-11-13T18:27:21,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441974 2024-11-13T18:27:21,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441974 2024-11-13T18:27:21,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441974) roll requested 2024-11-13T18:27:21,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441981 2024-11-13T18:27:21,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441981 2024-11-13T18:27:21,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441981) roll requested 2024-11-13T18:27:21,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441989 2024-11-13T18:27:21,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:21,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441989 2024-11-13T18:27:21,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:21,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:21,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441989) roll requested 2024-11-13T18:27:21,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522441996 2024-11-13T18:27:22,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441996 2024-11-13T18:27:22,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522441996) roll requested 2024-11-13T18:27:22,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442004 2024-11-13T18:27:22,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522441996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442004 2024-11-13T18:27:22,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442004) roll requested 2024-11-13T18:27:22,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442012 2024-11-13T18:27:22,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442012 2024-11-13T18:27:22,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442012) roll requested 2024-11-13T18:27:22,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442019 2024-11-13T18:27:22,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442019 2024-11-13T18:27:22,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442019) roll requested 2024-11-13T18:27:22,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442029 2024-11-13T18:27:22,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442029 2024-11-13T18:27:22,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442029) roll requested 2024-11-13T18:27:22,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442039 2024-11-13T18:27:22,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442039 2024-11-13T18:27:22,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442039) roll requested 2024-11-13T18:27:22,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442053 2024-11-13T18:27:22,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442053 2024-11-13T18:27:22,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442053) roll requested 2024-11-13T18:27:22,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442062 2024-11-13T18:27:22,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442062 2024-11-13T18:27:22,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442062) roll requested 2024-11-13T18:27:22,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442070 2024-11-13T18:27:22,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442070 2024-11-13T18:27:22,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442070) roll requested 2024-11-13T18:27:22,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442078 2024-11-13T18:27:22,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442078 2024-11-13T18:27:22,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442078) roll requested 2024-11-13T18:27:22,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442086 2024-11-13T18:27:22,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442086 2024-11-13T18:27:22,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442086) roll requested 2024-11-13T18:27:22,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442094 2024-11-13T18:27:22,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442094 2024-11-13T18:27:22,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442094) roll requested 2024-11-13T18:27:22,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442103 2024-11-13T18:27:22,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442103 2024-11-13T18:27:22,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442103) roll requested 2024-11-13T18:27:22,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442111 2024-11-13T18:27:22,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442111 2024-11-13T18:27:22,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442111) roll requested 2024-11-13T18:27:22,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442119 2024-11-13T18:27:22,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442119 2024-11-13T18:27:22,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442119) roll requested 2024-11-13T18:27:22,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442127 2024-11-13T18:27:22,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442127 2024-11-13T18:27:22,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442127) roll requested 2024-11-13T18:27:22,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442140 2024-11-13T18:27:22,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442140 2024-11-13T18:27:22,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442140) roll requested 2024-11-13T18:27:22,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442147 2024-11-13T18:27:22,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442147 2024-11-13T18:27:22,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442147) roll requested 2024-11-13T18:27:22,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442155 2024-11-13T18:27:22,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442155 2024-11-13T18:27:22,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442155) roll requested 2024-11-13T18:27:22,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442163 2024-11-13T18:27:22,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442163 2024-11-13T18:27:22,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442163) roll requested 2024-11-13T18:27:22,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442171 2024-11-13T18:27:22,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442171 2024-11-13T18:27:22,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442171) roll requested 2024-11-13T18:27:22,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442180 2024-11-13T18:27:22,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442180 2024-11-13T18:27:22,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442180) roll requested 2024-11-13T18:27:22,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442189 2024-11-13T18:27:22,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442189 2024-11-13T18:27:22,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442189) roll requested 2024-11-13T18:27:22,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442197 2024-11-13T18:27:22,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442197 2024-11-13T18:27:22,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442197) roll requested 2024-11-13T18:27:22,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442205 2024-11-13T18:27:22,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442205 2024-11-13T18:27:22,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442205) roll requested 2024-11-13T18:27:22,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442213 2024-11-13T18:27:22,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442213 2024-11-13T18:27:22,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442213) roll requested 2024-11-13T18:27:22,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442222 2024-11-13T18:27:22,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442222 2024-11-13T18:27:22,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442222) roll requested 2024-11-13T18:27:22,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442229 2024-11-13T18:27:22,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442229 2024-11-13T18:27:22,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442229) roll requested 2024-11-13T18:27:22,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442237 2024-11-13T18:27:22,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442237 2024-11-13T18:27:22,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442237) roll requested 2024-11-13T18:27:22,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442244 2024-11-13T18:27:22,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442244 2024-11-13T18:27:22,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442244) roll requested 2024-11-13T18:27:22,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442251 2024-11-13T18:27:22,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442251 2024-11-13T18:27:22,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442251) roll requested 2024-11-13T18:27:22,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442261 2024-11-13T18:27:22,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442261 2024-11-13T18:27:22,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442261) roll requested 2024-11-13T18:27:22,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442269 2024-11-13T18:27:22,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442269 2024-11-13T18:27:22,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442269) roll requested 2024-11-13T18:27:22,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442277 2024-11-13T18:27:22,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442277 2024-11-13T18:27:22,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442277) roll requested 2024-11-13T18:27:22,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442285 2024-11-13T18:27:22,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442285 2024-11-13T18:27:22,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442285) roll requested 2024-11-13T18:27:22,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442292 2024-11-13T18:27:22,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442292 2024-11-13T18:27:22,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442292) roll requested 2024-11-13T18:27:22,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442303 2024-11-13T18:27:22,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442303 2024-11-13T18:27:22,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442303) roll requested 2024-11-13T18:27:22,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442313 2024-11-13T18:27:22,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442313 2024-11-13T18:27:22,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442313) roll requested 2024-11-13T18:27:22,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442325 2024-11-13T18:27:22,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442325 2024-11-13T18:27:22,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442325) roll requested 2024-11-13T18:27:22,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442332 2024-11-13T18:27:22,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442332 2024-11-13T18:27:22,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442332) roll requested 2024-11-13T18:27:22,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442340 2024-11-13T18:27:22,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442340 2024-11-13T18:27:22,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442340) roll requested 2024-11-13T18:27:22,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442348 2024-11-13T18:27:22,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442348 2024-11-13T18:27:22,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442348) roll requested 2024-11-13T18:27:22,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442356 2024-11-13T18:27:22,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442356 2024-11-13T18:27:22,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442356) roll requested 2024-11-13T18:27:22,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442366 2024-11-13T18:27:22,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442366 2024-11-13T18:27:22,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442366) roll requested 2024-11-13T18:27:22,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442374 2024-11-13T18:27:22,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442374 2024-11-13T18:27:22,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442374) roll requested 2024-11-13T18:27:22,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442382 2024-11-13T18:27:22,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442382 2024-11-13T18:27:22,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442382) roll requested 2024-11-13T18:27:22,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442390 2024-11-13T18:27:22,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442390 2024-11-13T18:27:22,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442390) roll requested 2024-11-13T18:27:22,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442398 2024-11-13T18:27:22,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442398 2024-11-13T18:27:22,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442398) roll requested 2024-11-13T18:27:22,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442406 2024-11-13T18:27:22,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442406 2024-11-13T18:27:22,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442406) roll requested 2024-11-13T18:27:22,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442413 2024-11-13T18:27:22,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442413 2024-11-13T18:27:22,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442413) roll requested 2024-11-13T18:27:22,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442421 2024-11-13T18:27:22,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442421 2024-11-13T18:27:22,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442421) roll requested 2024-11-13T18:27:22,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442429 2024-11-13T18:27:22,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442429 2024-11-13T18:27:22,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442429) roll requested 2024-11-13T18:27:22,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442437 2024-11-13T18:27:22,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442437 2024-11-13T18:27:22,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442437) roll requested 2024-11-13T18:27:22,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442444 2024-11-13T18:27:22,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442444 2024-11-13T18:27:22,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442444) roll requested 2024-11-13T18:27:22,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442450 2024-11-13T18:27:22,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442450 2024-11-13T18:27:22,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442450) roll requested 2024-11-13T18:27:22,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442457 2024-11-13T18:27:22,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442457 2024-11-13T18:27:22,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442457) roll requested 2024-11-13T18:27:22,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442464 2024-11-13T18:27:22,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442464 2024-11-13T18:27:22,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442464) roll requested 2024-11-13T18:27:22,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442472 2024-11-13T18:27:22,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442472 2024-11-13T18:27:22,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442472) roll requested 2024-11-13T18:27:22,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442478 2024-11-13T18:27:22,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442478 2024-11-13T18:27:22,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442478) roll requested 2024-11-13T18:27:22,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442486 2024-11-13T18:27:22,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442486 2024-11-13T18:27:22,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442486) roll requested 2024-11-13T18:27:22,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442492 2024-11-13T18:27:22,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442492 2024-11-13T18:27:22,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442492) roll requested 2024-11-13T18:27:22,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442501 2024-11-13T18:27:22,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442501 2024-11-13T18:27:22,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442501) roll requested 2024-11-13T18:27:22,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442508 2024-11-13T18:27:22,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442508 2024-11-13T18:27:22,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442508) roll requested 2024-11-13T18:27:22,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442516 2024-11-13T18:27:22,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442516 2024-11-13T18:27:22,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442516) roll requested 2024-11-13T18:27:22,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442524 2024-11-13T18:27:22,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442524 2024-11-13T18:27:22,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442524) roll requested 2024-11-13T18:27:22,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442532 2024-11-13T18:27:22,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442532 2024-11-13T18:27:22,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442532) roll requested 2024-11-13T18:27:22,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442538 2024-11-13T18:27:22,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442538 2024-11-13T18:27:22,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442538) roll requested 2024-11-13T18:27:22,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442545 2024-11-13T18:27:22,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442545 2024-11-13T18:27:22,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442545) roll requested 2024-11-13T18:27:22,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442553 2024-11-13T18:27:22,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442553 2024-11-13T18:27:22,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442553) roll requested 2024-11-13T18:27:22,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442561 2024-11-13T18:27:22,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442561 2024-11-13T18:27:22,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442561) roll requested 2024-11-13T18:27:22,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442569 2024-11-13T18:27:22,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442569 2024-11-13T18:27:22,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442569) roll requested 2024-11-13T18:27:22,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442577 2024-11-13T18:27:22,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442577 2024-11-13T18:27:22,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442577) roll requested 2024-11-13T18:27:22,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442583 2024-11-13T18:27:22,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442583 2024-11-13T18:27:22,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442583) roll requested 2024-11-13T18:27:22,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442589 2024-11-13T18:27:22,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442589 2024-11-13T18:27:22,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442589) roll requested 2024-11-13T18:27:22,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442596 2024-11-13T18:27:22,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442596 2024-11-13T18:27:22,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442596) roll requested 2024-11-13T18:27:22,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442603 2024-11-13T18:27:22,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442603 2024-11-13T18:27:22,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442603) roll requested 2024-11-13T18:27:22,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442610 2024-11-13T18:27:22,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442610 2024-11-13T18:27:22,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442610) roll requested 2024-11-13T18:27:22,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442617 2024-11-13T18:27:22,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442617 2024-11-13T18:27:22,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442617) roll requested 2024-11-13T18:27:22,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442625 2024-11-13T18:27:22,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442625 2024-11-13T18:27:22,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442625) roll requested 2024-11-13T18:27:22,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442632 2024-11-13T18:27:22,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442632 2024-11-13T18:27:22,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442632) roll requested 2024-11-13T18:27:22,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442639 2024-11-13T18:27:22,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442639 2024-11-13T18:27:22,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442639) roll requested 2024-11-13T18:27:22,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442646 2024-11-13T18:27:22,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442646 2024-11-13T18:27:22,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442646) roll requested 2024-11-13T18:27:22,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442653 2024-11-13T18:27:22,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442653 2024-11-13T18:27:22,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442653) roll requested 2024-11-13T18:27:22,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442661 2024-11-13T18:27:22,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442661 2024-11-13T18:27:22,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442661) roll requested 2024-11-13T18:27:22,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442669 2024-11-13T18:27:22,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442669 2024-11-13T18:27:22,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442669) roll requested 2024-11-13T18:27:22,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442682 2024-11-13T18:27:22,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442682 2024-11-13T18:27:22,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442682) roll requested 2024-11-13T18:27:22,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442690 2024-11-13T18:27:22,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442690 2024-11-13T18:27:22,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442690) roll requested 2024-11-13T18:27:22,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442698 2024-11-13T18:27:22,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442698 2024-11-13T18:27:22,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442698) roll requested 2024-11-13T18:27:22,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442707 2024-11-13T18:27:22,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442707 2024-11-13T18:27:22,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442707) roll requested 2024-11-13T18:27:22,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442731 2024-11-13T18:27:22,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442731 2024-11-13T18:27:22,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442731) roll requested 2024-11-13T18:27:22,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442757 2024-11-13T18:27:22,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442757 2024-11-13T18:27:22,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442757) roll requested 2024-11-13T18:27:22,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442773 2024-11-13T18:27:22,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442773 2024-11-13T18:27:22,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442773) roll requested 2024-11-13T18:27:22,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442781 2024-11-13T18:27:22,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442781 2024-11-13T18:27:22,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442781) roll requested 2024-11-13T18:27:22,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442792 2024-11-13T18:27:22,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442792 2024-11-13T18:27:22,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442792) roll requested 2024-11-13T18:27:22,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442801 2024-11-13T18:27:22,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442801 2024-11-13T18:27:22,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442801) roll requested 2024-11-13T18:27:22,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442812 2024-11-13T18:27:22,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442812 2024-11-13T18:27:22,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442812) roll requested 2024-11-13T18:27:22,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442829 2024-11-13T18:27:22,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442829 2024-11-13T18:27:22,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442829) roll requested 2024-11-13T18:27:22,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442845 2024-11-13T18:27:22,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442845 2024-11-13T18:27:22,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442845) roll requested 2024-11-13T18:27:22,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442854 2024-11-13T18:27:22,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442854 2024-11-13T18:27:22,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442854) roll requested 2024-11-13T18:27:22,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442862 2024-11-13T18:27:22,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442862 2024-11-13T18:27:22,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442862) roll requested 2024-11-13T18:27:22,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442870 2024-11-13T18:27:22,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442870 2024-11-13T18:27:22,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442870) roll requested 2024-11-13T18:27:22,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442886 2024-11-13T18:27:22,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442886 2024-11-13T18:27:22,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442886) roll requested 2024-11-13T18:27:22,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442893 2024-11-13T18:27:22,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442893 2024-11-13T18:27:22,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442893) roll requested 2024-11-13T18:27:22,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442900 2024-11-13T18:27:22,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442900 2024-11-13T18:27:22,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442900) roll requested 2024-11-13T18:27:22,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442908 2024-11-13T18:27:22,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442908 2024-11-13T18:27:22,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442908) roll requested 2024-11-13T18:27:22,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442916 2024-11-13T18:27:22,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442916 2024-11-13T18:27:22,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442916) roll requested 2024-11-13T18:27:22,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442924 2024-11-13T18:27:22,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442924 2024-11-13T18:27:22,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442924) roll requested 2024-11-13T18:27:22,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442932 2024-11-13T18:27:22,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442932 2024-11-13T18:27:22,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442932) roll requested 2024-11-13T18:27:22,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442940 2024-11-13T18:27:22,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442940 2024-11-13T18:27:22,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442940) roll requested 2024-11-13T18:27:22,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442946 2024-11-13T18:27:22,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442946 2024-11-13T18:27:22,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442946) roll requested 2024-11-13T18:27:22,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442954 2024-11-13T18:27:22,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442954 2024-11-13T18:27:22,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442954) roll requested 2024-11-13T18:27:22,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442961 2024-11-13T18:27:22,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442961 2024-11-13T18:27:22,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442961) roll requested 2024-11-13T18:27:22,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442968 2024-11-13T18:27:22,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442968 2024-11-13T18:27:22,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442968) roll requested 2024-11-13T18:27:22,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442975 2024-11-13T18:27:22,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442975 2024-11-13T18:27:22,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442975) roll requested 2024-11-13T18:27:22,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442982 2024-11-13T18:27:22,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442982 2024-11-13T18:27:22,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442982) roll requested 2024-11-13T18:27:22,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442990 2024-11-13T18:27:22,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:22,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442990 2024-11-13T18:27:22,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:22,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:22,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442990) roll requested 2024-11-13T18:27:22,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522442997 2024-11-13T18:27:23,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442997 2024-11-13T18:27:23,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522442997) roll requested 2024-11-13T18:27:23,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443005 2024-11-13T18:27:23,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522442997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443005 2024-11-13T18:27:23,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443005) roll requested 2024-11-13T18:27:23,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443012 2024-11-13T18:27:23,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443012 2024-11-13T18:27:23,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443012) roll requested 2024-11-13T18:27:23,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443019 2024-11-13T18:27:23,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443019 2024-11-13T18:27:23,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443019) roll requested 2024-11-13T18:27:23,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443027 2024-11-13T18:27:23,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443027 2024-11-13T18:27:23,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443027) roll requested 2024-11-13T18:27:23,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443035 2024-11-13T18:27:23,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443035 2024-11-13T18:27:23,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443035) roll requested 2024-11-13T18:27:23,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443046 2024-11-13T18:27:23,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443046 2024-11-13T18:27:23,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443046) roll requested 2024-11-13T18:27:23,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443060 2024-11-13T18:27:23,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443060 2024-11-13T18:27:23,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443060) roll requested 2024-11-13T18:27:23,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443066 2024-11-13T18:27:23,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443066 2024-11-13T18:27:23,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443066) roll requested 2024-11-13T18:27:23,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443074 2024-11-13T18:27:23,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443074 2024-11-13T18:27:23,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443074) roll requested 2024-11-13T18:27:23,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443082 2024-11-13T18:27:23,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443082 2024-11-13T18:27:23,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443082) roll requested 2024-11-13T18:27:23,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443089 2024-11-13T18:27:23,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443089 2024-11-13T18:27:23,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443089) roll requested 2024-11-13T18:27:23,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443097 2024-11-13T18:27:23,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443097 2024-11-13T18:27:23,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443097) roll requested 2024-11-13T18:27:23,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443104 2024-11-13T18:27:23,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443104 2024-11-13T18:27:23,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443104) roll requested 2024-11-13T18:27:23,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443112 2024-11-13T18:27:23,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443112 2024-11-13T18:27:23,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443112) roll requested 2024-11-13T18:27:23,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443119 2024-11-13T18:27:23,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443119 2024-11-13T18:27:23,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443119) roll requested 2024-11-13T18:27:23,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443127 2024-11-13T18:27:23,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443127 2024-11-13T18:27:23,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443127) roll requested 2024-11-13T18:27:23,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443135 2024-11-13T18:27:23,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443135 2024-11-13T18:27:23,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443135) roll requested 2024-11-13T18:27:23,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443144 2024-11-13T18:27:23,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443144 2024-11-13T18:27:23,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443144) roll requested 2024-11-13T18:27:23,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443152 2024-11-13T18:27:23,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443152 2024-11-13T18:27:23,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443152) roll requested 2024-11-13T18:27:23,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443159 2024-11-13T18:27:23,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443159 2024-11-13T18:27:23,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443159) roll requested 2024-11-13T18:27:23,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443167 2024-11-13T18:27:23,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443167 2024-11-13T18:27:23,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443167) roll requested 2024-11-13T18:27:23,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443174 2024-11-13T18:27:23,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443174 2024-11-13T18:27:23,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443174) roll requested 2024-11-13T18:27:23,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443184 2024-11-13T18:27:23,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443184 2024-11-13T18:27:23,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443184) roll requested 2024-11-13T18:27:23,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443190 2024-11-13T18:27:23,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443190 2024-11-13T18:27:23,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443190) roll requested 2024-11-13T18:27:23,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443198 2024-11-13T18:27:23,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443198 2024-11-13T18:27:23,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443198) roll requested 2024-11-13T18:27:23,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443206 2024-11-13T18:27:23,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443206 2024-11-13T18:27:23,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443206) roll requested 2024-11-13T18:27:23,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443217 2024-11-13T18:27:23,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443217 2024-11-13T18:27:23,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443217) roll requested 2024-11-13T18:27:23,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443227 2024-11-13T18:27:23,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443227 2024-11-13T18:27:23,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443227) roll requested 2024-11-13T18:27:23,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443237 2024-11-13T18:27:23,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443237 2024-11-13T18:27:23,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443237) roll requested 2024-11-13T18:27:23,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443246 2024-11-13T18:27:23,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443246 2024-11-13T18:27:23,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443246) roll requested 2024-11-13T18:27:23,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443256 2024-11-13T18:27:23,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443256 2024-11-13T18:27:23,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443256) roll requested 2024-11-13T18:27:23,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443264 2024-11-13T18:27:23,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443264 2024-11-13T18:27:23,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443264) roll requested 2024-11-13T18:27:23,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443271 2024-11-13T18:27:23,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443271 2024-11-13T18:27:23,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443271) roll requested 2024-11-13T18:27:23,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443284 2024-11-13T18:27:23,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443284 2024-11-13T18:27:23,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443284) roll requested 2024-11-13T18:27:23,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443292 2024-11-13T18:27:23,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443292 2024-11-13T18:27:23,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443292) roll requested 2024-11-13T18:27:23,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443300 2024-11-13T18:27:23,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443300 2024-11-13T18:27:23,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443300) roll requested 2024-11-13T18:27:23,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443308 2024-11-13T18:27:23,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443308 2024-11-13T18:27:23,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443308) roll requested 2024-11-13T18:27:23,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443325 2024-11-13T18:27:23,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443325 2024-11-13T18:27:23,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443325) roll requested 2024-11-13T18:27:23,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443341 2024-11-13T18:27:23,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443341 2024-11-13T18:27:23,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443341) roll requested 2024-11-13T18:27:23,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443355 2024-11-13T18:27:23,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443355 2024-11-13T18:27:23,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443355) roll requested 2024-11-13T18:27:23,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443365 2024-11-13T18:27:23,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443365 2024-11-13T18:27:23,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443365) roll requested 2024-11-13T18:27:23,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443372 2024-11-13T18:27:23,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443372 2024-11-13T18:27:23,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443372) roll requested 2024-11-13T18:27:23,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443383 2024-11-13T18:27:23,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443383 2024-11-13T18:27:23,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443383) roll requested 2024-11-13T18:27:23,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443393 2024-11-13T18:27:23,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443393 2024-11-13T18:27:23,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443393) roll requested 2024-11-13T18:27:23,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443402 2024-11-13T18:27:23,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443402 2024-11-13T18:27:23,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443402) roll requested 2024-11-13T18:27:23,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443412 2024-11-13T18:27:23,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443412 2024-11-13T18:27:23,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443412) roll requested 2024-11-13T18:27:23,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443419 2024-11-13T18:27:23,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443419 2024-11-13T18:27:23,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443419) roll requested 2024-11-13T18:27:23,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443438 2024-11-13T18:27:23,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443438 2024-11-13T18:27:23,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443438) roll requested 2024-11-13T18:27:23,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443450 2024-11-13T18:27:23,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443450 2024-11-13T18:27:23,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443450) roll requested 2024-11-13T18:27:23,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443465 2024-11-13T18:27:23,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443465 2024-11-13T18:27:23,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443465) roll requested 2024-11-13T18:27:23,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443476 2024-11-13T18:27:23,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443476 2024-11-13T18:27:23,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443476) roll requested 2024-11-13T18:27:23,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443487 2024-11-13T18:27:23,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443487 2024-11-13T18:27:23,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443487) roll requested 2024-11-13T18:27:23,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443495 2024-11-13T18:27:23,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443495 2024-11-13T18:27:23,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443495) roll requested 2024-11-13T18:27:23,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443509 2024-11-13T18:27:23,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443509 2024-11-13T18:27:23,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443509) roll requested 2024-11-13T18:27:23,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443521 2024-11-13T18:27:23,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443521 2024-11-13T18:27:23,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443521) roll requested 2024-11-13T18:27:23,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443533 2024-11-13T18:27:23,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443533 2024-11-13T18:27:23,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443533) roll requested 2024-11-13T18:27:23,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443544 2024-11-13T18:27:23,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443544 2024-11-13T18:27:23,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443544) roll requested 2024-11-13T18:27:23,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443554 2024-11-13T18:27:23,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443554 2024-11-13T18:27:23,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443554) roll requested 2024-11-13T18:27:23,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443561 2024-11-13T18:27:23,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443561 2024-11-13T18:27:23,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443561) roll requested 2024-11-13T18:27:23,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443568 2024-11-13T18:27:23,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443568 2024-11-13T18:27:23,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443568) roll requested 2024-11-13T18:27:23,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443576 2024-11-13T18:27:23,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443576 2024-11-13T18:27:23,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443576) roll requested 2024-11-13T18:27:23,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443583 2024-11-13T18:27:23,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443583 2024-11-13T18:27:23,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443583) roll requested 2024-11-13T18:27:23,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443595 2024-11-13T18:27:23,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443595 2024-11-13T18:27:23,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443595) roll requested 2024-11-13T18:27:23,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443603 2024-11-13T18:27:23,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443603 2024-11-13T18:27:23,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443603) roll requested 2024-11-13T18:27:23,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443610 2024-11-13T18:27:23,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443610 2024-11-13T18:27:23,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443610) roll requested 2024-11-13T18:27:23,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443617 2024-11-13T18:27:23,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443617 2024-11-13T18:27:23,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443617) roll requested 2024-11-13T18:27:23,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443625 2024-11-13T18:27:23,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443625 2024-11-13T18:27:23,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443625) roll requested 2024-11-13T18:27:23,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443631 2024-11-13T18:27:23,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443631 2024-11-13T18:27:23,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443631) roll requested 2024-11-13T18:27:23,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443639 2024-11-13T18:27:23,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443639 2024-11-13T18:27:23,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443639) roll requested 2024-11-13T18:27:23,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443649 2024-11-13T18:27:23,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443649 2024-11-13T18:27:23,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443649) roll requested 2024-11-13T18:27:23,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443657 2024-11-13T18:27:23,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443657 2024-11-13T18:27:23,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443657) roll requested 2024-11-13T18:27:23,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443669 2024-11-13T18:27:23,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443669 2024-11-13T18:27:23,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443669) roll requested 2024-11-13T18:27:23,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443678 2024-11-13T18:27:23,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443678 2024-11-13T18:27:23,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443678) roll requested 2024-11-13T18:27:23,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443688 2024-11-13T18:27:23,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443688 2024-11-13T18:27:23,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443688) roll requested 2024-11-13T18:27:23,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443701 2024-11-13T18:27:23,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443701 2024-11-13T18:27:23,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443701) roll requested 2024-11-13T18:27:23,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443709 2024-11-13T18:27:23,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443709 2024-11-13T18:27:23,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443709) roll requested 2024-11-13T18:27:23,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443718 2024-11-13T18:27:23,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443718 2024-11-13T18:27:23,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443718) roll requested 2024-11-13T18:27:23,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443726 2024-11-13T18:27:23,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443726 2024-11-13T18:27:23,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443726) roll requested 2024-11-13T18:27:23,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443733 2024-11-13T18:27:23,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443733 2024-11-13T18:27:23,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443733) roll requested 2024-11-13T18:27:23,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443741 2024-11-13T18:27:23,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443741 2024-11-13T18:27:23,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443741) roll requested 2024-11-13T18:27:23,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443749 2024-11-13T18:27:23,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443749 2024-11-13T18:27:23,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443749) roll requested 2024-11-13T18:27:23,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443757 2024-11-13T18:27:23,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443757 2024-11-13T18:27:23,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443757) roll requested 2024-11-13T18:27:23,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443764 2024-11-13T18:27:23,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443764 2024-11-13T18:27:23,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443764) roll requested 2024-11-13T18:27:23,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443772 2024-11-13T18:27:23,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443772 2024-11-13T18:27:23,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443772) roll requested 2024-11-13T18:27:23,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443780 2024-11-13T18:27:23,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443780 2024-11-13T18:27:23,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443780) roll requested 2024-11-13T18:27:23,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443786 2024-11-13T18:27:23,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443786 2024-11-13T18:27:23,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443786) roll requested 2024-11-13T18:27:23,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443794 2024-11-13T18:27:23,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443794 2024-11-13T18:27:23,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443794) roll requested 2024-11-13T18:27:23,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443806 2024-11-13T18:27:23,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443806 2024-11-13T18:27:23,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443806) roll requested 2024-11-13T18:27:23,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443813 2024-11-13T18:27:23,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443813 2024-11-13T18:27:23,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443813) roll requested 2024-11-13T18:27:23,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443821 2024-11-13T18:27:23,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443821 2024-11-13T18:27:23,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443821) roll requested 2024-11-13T18:27:23,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443829 2024-11-13T18:27:23,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443829 2024-11-13T18:27:23,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443821 is not closed yet, will try archiving it next time 2024-11-13T18:27:23,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443829) roll requested 2024-11-13T18:27:23,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443836 2024-11-13T18:27:23,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443836 2024-11-13T18:27:23,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443836) roll requested 2024-11-13T18:27:23,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443846 2024-11-13T18:27:23,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443846 2024-11-13T18:27:23,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443846) roll requested 2024-11-13T18:27:23,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443854 2024-11-13T18:27:23,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443854 2024-11-13T18:27:23,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443854) roll requested 2024-11-13T18:27:23,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443866 2024-11-13T18:27:23,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443866 2024-11-13T18:27:23,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443866) roll requested 2024-11-13T18:27:23,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443881 2024-11-13T18:27:23,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443881 2024-11-13T18:27:23,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443881) roll requested 2024-11-13T18:27:23,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443894 2024-11-13T18:27:23,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443894 2024-11-13T18:27:23,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443894) roll requested 2024-11-13T18:27:23,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443906 2024-11-13T18:27:23,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443906 2024-11-13T18:27:23,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443906) roll requested 2024-11-13T18:27:23,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443921 2024-11-13T18:27:23,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443921 2024-11-13T18:27:23,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443921) roll requested 2024-11-13T18:27:23,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443933 2024-11-13T18:27:23,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443933 2024-11-13T18:27:23,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443933) roll requested 2024-11-13T18:27:23,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443942 2024-11-13T18:27:23,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443942 2024-11-13T18:27:23,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443942) roll requested 2024-11-13T18:27:23,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443950 2024-11-13T18:27:23,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443950 2024-11-13T18:27:23,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443950) roll requested 2024-11-13T18:27:23,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443958 2024-11-13T18:27:23,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443958 2024-11-13T18:27:23,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443958) roll requested 2024-11-13T18:27:23,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443965 2024-11-13T18:27:23,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443965 2024-11-13T18:27:23,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443965) roll requested 2024-11-13T18:27:23,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443972 2024-11-13T18:27:23,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443972 2024-11-13T18:27:23,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443972) roll requested 2024-11-13T18:27:23,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443979 2024-11-13T18:27:23,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443979 2024-11-13T18:27:23,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443979) roll requested 2024-11-13T18:27:23,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443986 2024-11-13T18:27:23,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:23,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443986 2024-11-13T18:27:23,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:23,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:23,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443986) roll requested 2024-11-13T18:27:23,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522443994 2024-11-13T18:27:24,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443994 2024-11-13T18:27:24,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522443994) roll requested 2024-11-13T18:27:24,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444001 2024-11-13T18:27:24,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522443994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444001 2024-11-13T18:27:24,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444001) roll requested 2024-11-13T18:27:24,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444008 2024-11-13T18:27:24,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444008 2024-11-13T18:27:24,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444008) roll requested 2024-11-13T18:27:24,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444016 2024-11-13T18:27:24,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444016 2024-11-13T18:27:24,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444016) roll requested 2024-11-13T18:27:24,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444023 2024-11-13T18:27:24,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444023 2024-11-13T18:27:24,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444023) roll requested 2024-11-13T18:27:24,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444034 2024-11-13T18:27:24,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444034 2024-11-13T18:27:24,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444034) roll requested 2024-11-13T18:27:24,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444043 2024-11-13T18:27:24,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444043 2024-11-13T18:27:24,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444043) roll requested 2024-11-13T18:27:24,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444051 2024-11-13T18:27:24,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444051 2024-11-13T18:27:24,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444051) roll requested 2024-11-13T18:27:24,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444058 2024-11-13T18:27:24,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444058 2024-11-13T18:27:24,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444058) roll requested 2024-11-13T18:27:24,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444065 2024-11-13T18:27:24,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444065 2024-11-13T18:27:24,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444065) roll requested 2024-11-13T18:27:24,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444072 2024-11-13T18:27:24,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444072 2024-11-13T18:27:24,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444072) roll requested 2024-11-13T18:27:24,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444080 2024-11-13T18:27:24,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444080 2024-11-13T18:27:24,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444080) roll requested 2024-11-13T18:27:24,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444089 2024-11-13T18:27:24,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444089 2024-11-13T18:27:24,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444089) roll requested 2024-11-13T18:27:24,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444097 2024-11-13T18:27:24,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444097 2024-11-13T18:27:24,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444097) roll requested 2024-11-13T18:27:24,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444104 2024-11-13T18:27:24,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444104 2024-11-13T18:27:24,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444104) roll requested 2024-11-13T18:27:24,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444110 2024-11-13T18:27:24,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444110 2024-11-13T18:27:24,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444110) roll requested 2024-11-13T18:27:24,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444118 2024-11-13T18:27:24,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444118 2024-11-13T18:27:24,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444118) roll requested 2024-11-13T18:27:24,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444124 2024-11-13T18:27:24,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444124 2024-11-13T18:27:24,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444124) roll requested 2024-11-13T18:27:24,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444131 2024-11-13T18:27:24,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444131 2024-11-13T18:27:24,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444131) roll requested 2024-11-13T18:27:24,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444139 2024-11-13T18:27:24,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444139 2024-11-13T18:27:24,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444139) roll requested 2024-11-13T18:27:24,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444146 2024-11-13T18:27:24,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444146 2024-11-13T18:27:24,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444146) roll requested 2024-11-13T18:27:24,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444153 2024-11-13T18:27:24,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444153 2024-11-13T18:27:24,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444153) roll requested 2024-11-13T18:27:24,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444161 2024-11-13T18:27:24,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444161 2024-11-13T18:27:24,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444161) roll requested 2024-11-13T18:27:24,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444169 2024-11-13T18:27:24,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444169 2024-11-13T18:27:24,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444169) roll requested 2024-11-13T18:27:24,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444176 2024-11-13T18:27:24,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444176 2024-11-13T18:27:24,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444176) roll requested 2024-11-13T18:27:24,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444184 2024-11-13T18:27:24,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444184 2024-11-13T18:27:24,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444184) roll requested 2024-11-13T18:27:24,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444192 2024-11-13T18:27:24,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444192 2024-11-13T18:27:24,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444192) roll requested 2024-11-13T18:27:24,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444200 2024-11-13T18:27:24,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444200 2024-11-13T18:27:24,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444200) roll requested 2024-11-13T18:27:24,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444207 2024-11-13T18:27:24,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444207 2024-11-13T18:27:24,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444207) roll requested 2024-11-13T18:27:24,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444215 2024-11-13T18:27:24,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444215 2024-11-13T18:27:24,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444215) roll requested 2024-11-13T18:27:24,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444222 2024-11-13T18:27:24,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444222 2024-11-13T18:27:24,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444222) roll requested 2024-11-13T18:27:24,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444230 2024-11-13T18:27:24,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444230 2024-11-13T18:27:24,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444230) roll requested 2024-11-13T18:27:24,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444237 2024-11-13T18:27:24,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444237 2024-11-13T18:27:24,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444237) roll requested 2024-11-13T18:27:24,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444245 2024-11-13T18:27:24,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444245 2024-11-13T18:27:24,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444245) roll requested 2024-11-13T18:27:24,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444252 2024-11-13T18:27:24,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444252 2024-11-13T18:27:24,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444252) roll requested 2024-11-13T18:27:24,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444260 2024-11-13T18:27:24,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444260 2024-11-13T18:27:24,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444260) roll requested 2024-11-13T18:27:24,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444267 2024-11-13T18:27:24,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444267 2024-11-13T18:27:24,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444267) roll requested 2024-11-13T18:27:24,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444275 2024-11-13T18:27:24,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444275 2024-11-13T18:27:24,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444275) roll requested 2024-11-13T18:27:24,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444289 2024-11-13T18:27:24,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444289 2024-11-13T18:27:24,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444289) roll requested 2024-11-13T18:27:24,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444297 2024-11-13T18:27:24,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444297 2024-11-13T18:27:24,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444297) roll requested 2024-11-13T18:27:24,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444305 2024-11-13T18:27:24,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444305 2024-11-13T18:27:24,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444305) roll requested 2024-11-13T18:27:24,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444313 2024-11-13T18:27:24,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444313 2024-11-13T18:27:24,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444313) roll requested 2024-11-13T18:27:24,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444321 2024-11-13T18:27:24,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444321 2024-11-13T18:27:24,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444321) roll requested 2024-11-13T18:27:24,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444329 2024-11-13T18:27:24,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444329 2024-11-13T18:27:24,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444329) roll requested 2024-11-13T18:27:24,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444336 2024-11-13T18:27:24,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444336 2024-11-13T18:27:24,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444336) roll requested 2024-11-13T18:27:24,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444344 2024-11-13T18:27:24,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444344 2024-11-13T18:27:24,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444344) roll requested 2024-11-13T18:27:24,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444353 2024-11-13T18:27:24,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444353 2024-11-13T18:27:24,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444353) roll requested 2024-11-13T18:27:24,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444361 2024-11-13T18:27:24,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444361 2024-11-13T18:27:24,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444361) roll requested 2024-11-13T18:27:24,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444372 2024-11-13T18:27:24,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444372 2024-11-13T18:27:24,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444372) roll requested 2024-11-13T18:27:24,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444385 2024-11-13T18:27:24,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444385 2024-11-13T18:27:24,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444385) roll requested 2024-11-13T18:27:24,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444393 2024-11-13T18:27:24,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444393 2024-11-13T18:27:24,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444393) roll requested 2024-11-13T18:27:24,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444403 2024-11-13T18:27:24,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444403 2024-11-13T18:27:24,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444403) roll requested 2024-11-13T18:27:24,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444411 2024-11-13T18:27:24,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444411 2024-11-13T18:27:24,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444411) roll requested 2024-11-13T18:27:24,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444419 2024-11-13T18:27:24,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444419 2024-11-13T18:27:24,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444419) roll requested 2024-11-13T18:27:24,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444427 2024-11-13T18:27:24,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444427 2024-11-13T18:27:24,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444427) roll requested 2024-11-13T18:27:24,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444434 2024-11-13T18:27:24,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444434 2024-11-13T18:27:24,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444434) roll requested 2024-11-13T18:27:24,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444443 2024-11-13T18:27:24,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444443 2024-11-13T18:27:24,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444443) roll requested 2024-11-13T18:27:24,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444450 2024-11-13T18:27:24,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444450 2024-11-13T18:27:24,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444450) roll requested 2024-11-13T18:27:24,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444458 2024-11-13T18:27:24,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444458 2024-11-13T18:27:24,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444458) roll requested 2024-11-13T18:27:24,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444465 2024-11-13T18:27:24,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444465 2024-11-13T18:27:24,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444465) roll requested 2024-11-13T18:27:24,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444472 2024-11-13T18:27:24,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444472 2024-11-13T18:27:24,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444472) roll requested 2024-11-13T18:27:24,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444480 2024-11-13T18:27:24,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444480 2024-11-13T18:27:24,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444480) roll requested 2024-11-13T18:27:24,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444489 2024-11-13T18:27:24,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444489 2024-11-13T18:27:24,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444489) roll requested 2024-11-13T18:27:24,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444498 2024-11-13T18:27:24,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444498 2024-11-13T18:27:24,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444498) roll requested 2024-11-13T18:27:24,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444505 2024-11-13T18:27:24,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444505 2024-11-13T18:27:24,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444505) roll requested 2024-11-13T18:27:24,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444517 2024-11-13T18:27:24,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444517 2024-11-13T18:27:24,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444517) roll requested 2024-11-13T18:27:24,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444524 2024-11-13T18:27:24,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444524 2024-11-13T18:27:24,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444524) roll requested 2024-11-13T18:27:24,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444532 2024-11-13T18:27:24,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444532 2024-11-13T18:27:24,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444532) roll requested 2024-11-13T18:27:24,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444541 2024-11-13T18:27:24,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444541 2024-11-13T18:27:24,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444541) roll requested 2024-11-13T18:27:24,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444549 2024-11-13T18:27:24,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444549 2024-11-13T18:27:24,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444549) roll requested 2024-11-13T18:27:24,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444556 2024-11-13T18:27:24,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444556 2024-11-13T18:27:24,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444556) roll requested 2024-11-13T18:27:24,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444563 2024-11-13T18:27:24,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444563 2024-11-13T18:27:24,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444563) roll requested 2024-11-13T18:27:24,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444571 2024-11-13T18:27:24,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444571 2024-11-13T18:27:24,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444571) roll requested 2024-11-13T18:27:24,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444578 2024-11-13T18:27:24,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444578 2024-11-13T18:27:24,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444578) roll requested 2024-11-13T18:27:24,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444585 2024-11-13T18:27:24,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444585 2024-11-13T18:27:24,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444585) roll requested 2024-11-13T18:27:24,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444593 2024-11-13T18:27:24,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444593 2024-11-13T18:27:24,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444593) roll requested 2024-11-13T18:27:24,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444600 2024-11-13T18:27:24,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444600 2024-11-13T18:27:24,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444600) roll requested 2024-11-13T18:27:24,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444608 2024-11-13T18:27:24,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444608 2024-11-13T18:27:24,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444608) roll requested 2024-11-13T18:27:24,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444616 2024-11-13T18:27:24,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444616 2024-11-13T18:27:24,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444616) roll requested 2024-11-13T18:27:24,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444624 2024-11-13T18:27:24,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444624 2024-11-13T18:27:24,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444624) roll requested 2024-11-13T18:27:24,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444632 2024-11-13T18:27:24,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444632 2024-11-13T18:27:24,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444632) roll requested 2024-11-13T18:27:24,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444639 2024-11-13T18:27:24,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444639 2024-11-13T18:27:24,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444639) roll requested 2024-11-13T18:27:24,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444647 2024-11-13T18:27:24,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444647 2024-11-13T18:27:24,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444647) roll requested 2024-11-13T18:27:24,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444658 2024-11-13T18:27:24,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444658 2024-11-13T18:27:24,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444658) roll requested 2024-11-13T18:27:24,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444665 2024-11-13T18:27:24,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444665 2024-11-13T18:27:24,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444665) roll requested 2024-11-13T18:27:24,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444676 2024-11-13T18:27:24,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444676 2024-11-13T18:27:24,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444676) roll requested 2024-11-13T18:27:24,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444685 2024-11-13T18:27:24,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444685 2024-11-13T18:27:24,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444685) roll requested 2024-11-13T18:27:24,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444720 2024-11-13T18:27:24,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444720 2024-11-13T18:27:24,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444720) roll requested 2024-11-13T18:27:24,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444742 2024-11-13T18:27:24,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444742 2024-11-13T18:27:24,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444742) roll requested 2024-11-13T18:27:24,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444753 2024-11-13T18:27:24,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444753 2024-11-13T18:27:24,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444753) roll requested 2024-11-13T18:27:24,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444759 2024-11-13T18:27:24,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444759 2024-11-13T18:27:24,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444759) roll requested 2024-11-13T18:27:24,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444766 2024-11-13T18:27:24,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444766 2024-11-13T18:27:24,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444766) roll requested 2024-11-13T18:27:24,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444772 2024-11-13T18:27:24,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444772 2024-11-13T18:27:24,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444772) roll requested 2024-11-13T18:27:24,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444779 2024-11-13T18:27:24,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444779 2024-11-13T18:27:24,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444779) roll requested 2024-11-13T18:27:24,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444785 2024-11-13T18:27:24,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444785 2024-11-13T18:27:24,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444785) roll requested 2024-11-13T18:27:24,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444791 2024-11-13T18:27:24,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444791 2024-11-13T18:27:24,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444791) roll requested 2024-11-13T18:27:24,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444798 2024-11-13T18:27:24,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444798 2024-11-13T18:27:24,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444798) roll requested 2024-11-13T18:27:24,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444805 2024-11-13T18:27:24,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444805 2024-11-13T18:27:24,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444805) roll requested 2024-11-13T18:27:24,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444812 2024-11-13T18:27:24,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444812 2024-11-13T18:27:24,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444812) roll requested 2024-11-13T18:27:24,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444818 2024-11-13T18:27:24,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444818 2024-11-13T18:27:24,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444818) roll requested 2024-11-13T18:27:24,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444825 2024-11-13T18:27:24,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444825 2024-11-13T18:27:24,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444825) roll requested 2024-11-13T18:27:24,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444835 2024-11-13T18:27:24,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444835 2024-11-13T18:27:24,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444835) roll requested 2024-11-13T18:27:24,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444844 2024-11-13T18:27:24,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444844 2024-11-13T18:27:24,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444844) roll requested 2024-11-13T18:27:24,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444861 2024-11-13T18:27:24,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444861 2024-11-13T18:27:24,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444861) roll requested 2024-11-13T18:27:24,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444878 2024-11-13T18:27:24,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444878 2024-11-13T18:27:24,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444878) roll requested 2024-11-13T18:27:24,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444888 2024-11-13T18:27:24,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444888 2024-11-13T18:27:24,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444888) roll requested 2024-11-13T18:27:24,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444901 2024-11-13T18:27:24,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444901 2024-11-13T18:27:24,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444901) roll requested 2024-11-13T18:27:24,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444921 2024-11-13T18:27:24,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444921 2024-11-13T18:27:24,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444921) roll requested 2024-11-13T18:27:24,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444934 2024-11-13T18:27:24,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444934 2024-11-13T18:27:24,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444934) roll requested 2024-11-13T18:27:24,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444943 2024-11-13T18:27:24,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444943 2024-11-13T18:27:24,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444943) roll requested 2024-11-13T18:27:24,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444951 2024-11-13T18:27:24,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444951 2024-11-13T18:27:24,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444951) roll requested 2024-11-13T18:27:24,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444959 2024-11-13T18:27:24,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444959 2024-11-13T18:27:24,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444959) roll requested 2024-11-13T18:27:24,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444966 2024-11-13T18:27:24,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444966 2024-11-13T18:27:24,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444966) roll requested 2024-11-13T18:27:24,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444974 2024-11-13T18:27:24,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444974 2024-11-13T18:27:24,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444974) roll requested 2024-11-13T18:27:24,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444982 2024-11-13T18:27:24,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444982 2024-11-13T18:27:24,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444982) roll requested 2024-11-13T18:27:24,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444989 2024-11-13T18:27:24,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:24,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444989 2024-11-13T18:27:24,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:24,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:24,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444989) roll requested 2024-11-13T18:27:24,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522444997 2024-11-13T18:27:25,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444997 2024-11-13T18:27:25,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522444997) roll requested 2024-11-13T18:27:25,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445004 2024-11-13T18:27:25,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522444997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445004 2024-11-13T18:27:25,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445004) roll requested 2024-11-13T18:27:25,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445011 2024-11-13T18:27:25,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445011 2024-11-13T18:27:25,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445011) roll requested 2024-11-13T18:27:25,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445018 2024-11-13T18:27:25,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445018 2024-11-13T18:27:25,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445018) roll requested 2024-11-13T18:27:25,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445025 2024-11-13T18:27:25,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445025 2024-11-13T18:27:25,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445025) roll requested 2024-11-13T18:27:25,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445033 2024-11-13T18:27:25,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445033 2024-11-13T18:27:25,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445033) roll requested 2024-11-13T18:27:25,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445041 2024-11-13T18:27:25,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445041 2024-11-13T18:27:25,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445041) roll requested 2024-11-13T18:27:25,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445049 2024-11-13T18:27:25,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445049 2024-11-13T18:27:25,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445049) roll requested 2024-11-13T18:27:25,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445057 2024-11-13T18:27:25,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445057 2024-11-13T18:27:25,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445057) roll requested 2024-11-13T18:27:25,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445073 2024-11-13T18:27:25,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445073 2024-11-13T18:27:25,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445073) roll requested 2024-11-13T18:27:25,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445086 2024-11-13T18:27:25,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445086 2024-11-13T18:27:25,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445086) roll requested 2024-11-13T18:27:25,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445097 2024-11-13T18:27:25,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445097 2024-11-13T18:27:25,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445097) roll requested 2024-11-13T18:27:25,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445109 2024-11-13T18:27:25,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445109 2024-11-13T18:27:25,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445109) roll requested 2024-11-13T18:27:25,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445118 2024-11-13T18:27:25,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445118 2024-11-13T18:27:25,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445118) roll requested 2024-11-13T18:27:25,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445133 2024-11-13T18:27:25,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445133 2024-11-13T18:27:25,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445133) roll requested 2024-11-13T18:27:25,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445145 2024-11-13T18:27:25,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445145 2024-11-13T18:27:25,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445145) roll requested 2024-11-13T18:27:25,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445157 2024-11-13T18:27:25,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445157 2024-11-13T18:27:25,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445157) roll requested 2024-11-13T18:27:25,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445177 2024-11-13T18:27:25,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445177 2024-11-13T18:27:25,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445177) roll requested 2024-11-13T18:27:25,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445190 2024-11-13T18:27:25,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445190 2024-11-13T18:27:25,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445190) roll requested 2024-11-13T18:27:25,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445205 2024-11-13T18:27:25,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445205 2024-11-13T18:27:25,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445205) roll requested 2024-11-13T18:27:25,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445214 2024-11-13T18:27:25,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445214 2024-11-13T18:27:25,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445214) roll requested 2024-11-13T18:27:25,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445229 2024-11-13T18:27:25,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445229 2024-11-13T18:27:25,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445229) roll requested 2024-11-13T18:27:25,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445238 2024-11-13T18:27:25,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445238 2024-11-13T18:27:25,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445238) roll requested 2024-11-13T18:27:25,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445285 2024-11-13T18:27:25,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445285 2024-11-13T18:27:25,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445285) roll requested 2024-11-13T18:27:25,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445300 2024-11-13T18:27:25,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445300 2024-11-13T18:27:25,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445300) roll requested 2024-11-13T18:27:25,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445308 2024-11-13T18:27:25,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445308 2024-11-13T18:27:25,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445308) roll requested 2024-11-13T18:27:25,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445336 2024-11-13T18:27:25,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445336 2024-11-13T18:27:25,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445336) roll requested 2024-11-13T18:27:25,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445345 2024-11-13T18:27:25,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445345 2024-11-13T18:27:25,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445345) roll requested 2024-11-13T18:27:25,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445355 2024-11-13T18:27:25,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445355 2024-11-13T18:27:25,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445355) roll requested 2024-11-13T18:27:25,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445365 2024-11-13T18:27:25,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445365 2024-11-13T18:27:25,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445365) roll requested 2024-11-13T18:27:25,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445371 2024-11-13T18:27:25,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445371 2024-11-13T18:27:25,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445371) roll requested 2024-11-13T18:27:25,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445378 2024-11-13T18:27:25,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445378 2024-11-13T18:27:25,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445378) roll requested 2024-11-13T18:27:25,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445384 2024-11-13T18:27:25,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445384 2024-11-13T18:27:25,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445384) roll requested 2024-11-13T18:27:25,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445390 2024-11-13T18:27:25,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445390 2024-11-13T18:27:25,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445390) roll requested 2024-11-13T18:27:25,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445397 2024-11-13T18:27:25,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445397 2024-11-13T18:27:25,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445397) roll requested 2024-11-13T18:27:25,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445404 2024-11-13T18:27:25,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445404 2024-11-13T18:27:25,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445404) roll requested 2024-11-13T18:27:25,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445411 2024-11-13T18:27:25,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445411 2024-11-13T18:27:25,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445411) roll requested 2024-11-13T18:27:25,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445418 2024-11-13T18:27:25,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445418 2024-11-13T18:27:25,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445418) roll requested 2024-11-13T18:27:25,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445425 2024-11-13T18:27:25,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445425 2024-11-13T18:27:25,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445425) roll requested 2024-11-13T18:27:25,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445432 2024-11-13T18:27:25,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445432 2024-11-13T18:27:25,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445432) roll requested 2024-11-13T18:27:25,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445440 2024-11-13T18:27:25,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445440 2024-11-13T18:27:25,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445440) roll requested 2024-11-13T18:27:25,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445456 2024-11-13T18:27:25,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445456 2024-11-13T18:27:25,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445456) roll requested 2024-11-13T18:27:25,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445467 2024-11-13T18:27:25,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445467 2024-11-13T18:27:25,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445467) roll requested 2024-11-13T18:27:25,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445476 2024-11-13T18:27:25,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445476 2024-11-13T18:27:25,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445476) roll requested 2024-11-13T18:27:25,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445493 2024-11-13T18:27:25,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445493 2024-11-13T18:27:25,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445493) roll requested 2024-11-13T18:27:25,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445501 2024-11-13T18:27:25,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445501 2024-11-13T18:27:25,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445501) roll requested 2024-11-13T18:27:25,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445508 2024-11-13T18:27:25,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445508 2024-11-13T18:27:25,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445508) roll requested 2024-11-13T18:27:25,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445514 2024-11-13T18:27:25,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445514 2024-11-13T18:27:25,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445514) roll requested 2024-11-13T18:27:25,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445522 2024-11-13T18:27:25,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445522 2024-11-13T18:27:25,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445522) roll requested 2024-11-13T18:27:25,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445531 2024-11-13T18:27:25,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445531 2024-11-13T18:27:25,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445531) roll requested 2024-11-13T18:27:25,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445540 2024-11-13T18:27:25,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445540 2024-11-13T18:27:25,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445540) roll requested 2024-11-13T18:27:25,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445548 2024-11-13T18:27:25,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445548 2024-11-13T18:27:25,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445548) roll requested 2024-11-13T18:27:25,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445560 2024-11-13T18:27:25,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445560 2024-11-13T18:27:25,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445560) roll requested 2024-11-13T18:27:25,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445567 2024-11-13T18:27:25,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445567 2024-11-13T18:27:25,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445567) roll requested 2024-11-13T18:27:25,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445578 2024-11-13T18:27:25,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445578 2024-11-13T18:27:25,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445578) roll requested 2024-11-13T18:27:25,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445595 2024-11-13T18:27:25,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445595 2024-11-13T18:27:25,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445595) roll requested 2024-11-13T18:27:25,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445609 2024-11-13T18:27:25,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445609 2024-11-13T18:27:25,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445609) roll requested 2024-11-13T18:27:25,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445617 2024-11-13T18:27:25,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445617 2024-11-13T18:27:25,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445617) roll requested 2024-11-13T18:27:25,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445629 2024-11-13T18:27:25,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445629 2024-11-13T18:27:25,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445629) roll requested 2024-11-13T18:27:25,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445646 2024-11-13T18:27:25,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445646 2024-11-13T18:27:25,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445646) roll requested 2024-11-13T18:27:25,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445661 2024-11-13T18:27:25,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445661 2024-11-13T18:27:25,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445661) roll requested 2024-11-13T18:27:25,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445687 2024-11-13T18:27:25,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445687 2024-11-13T18:27:25,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445687) roll requested 2024-11-13T18:27:25,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445699 2024-11-13T18:27:25,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445699 2024-11-13T18:27:25,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445699) roll requested 2024-11-13T18:27:25,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445713 2024-11-13T18:27:25,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445713 2024-11-13T18:27:25,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445713) roll requested 2024-11-13T18:27:25,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445729 2024-11-13T18:27:25,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445729 2024-11-13T18:27:25,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445729) roll requested 2024-11-13T18:27:25,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445736 2024-11-13T18:27:25,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445736 2024-11-13T18:27:25,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445736) roll requested 2024-11-13T18:27:25,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445746 2024-11-13T18:27:25,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445746 2024-11-13T18:27:25,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445746) roll requested 2024-11-13T18:27:25,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445752 2024-11-13T18:27:25,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445752 2024-11-13T18:27:25,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445752) roll requested 2024-11-13T18:27:25,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445769 2024-11-13T18:27:25,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445769 2024-11-13T18:27:25,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445769) roll requested 2024-11-13T18:27:25,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445777 2024-11-13T18:27:25,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445777 2024-11-13T18:27:25,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445777) roll requested 2024-11-13T18:27:25,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445788 2024-11-13T18:27:25,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445788 2024-11-13T18:27:25,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445788) roll requested 2024-11-13T18:27:25,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445798 2024-11-13T18:27:25,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445798 2024-11-13T18:27:25,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445798) roll requested 2024-11-13T18:27:25,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445805 2024-11-13T18:27:25,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445805 2024-11-13T18:27:25,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445805) roll requested 2024-11-13T18:27:25,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445812 2024-11-13T18:27:25,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445812 2024-11-13T18:27:25,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445812) roll requested 2024-11-13T18:27:25,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445818 2024-11-13T18:27:25,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445818 2024-11-13T18:27:25,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445818) roll requested 2024-11-13T18:27:25,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445825 2024-11-13T18:27:25,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445825 2024-11-13T18:27:25,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445825) roll requested 2024-11-13T18:27:25,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445831 2024-11-13T18:27:25,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445831 2024-11-13T18:27:25,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445831) roll requested 2024-11-13T18:27:25,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445840 2024-11-13T18:27:25,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445840 2024-11-13T18:27:25,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445840) roll requested 2024-11-13T18:27:25,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445849 2024-11-13T18:27:25,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445849 2024-11-13T18:27:25,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445849) roll requested 2024-11-13T18:27:25,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445858 2024-11-13T18:27:25,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445858 2024-11-13T18:27:25,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445858) roll requested 2024-11-13T18:27:25,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445873 2024-11-13T18:27:25,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445873 2024-11-13T18:27:25,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445873) roll requested 2024-11-13T18:27:25,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445881 2024-11-13T18:27:25,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445881 2024-11-13T18:27:25,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445881) roll requested 2024-11-13T18:27:25,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445892 2024-11-13T18:27:25,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445892 2024-11-13T18:27:25,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445892) roll requested 2024-11-13T18:27:25,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445901 2024-11-13T18:27:25,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445901 2024-11-13T18:27:25,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445901) roll requested 2024-11-13T18:27:25,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445909 2024-11-13T18:27:25,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445909 2024-11-13T18:27:25,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445909) roll requested 2024-11-13T18:27:25,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445920 2024-11-13T18:27:25,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445920 2024-11-13T18:27:25,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445920) roll requested 2024-11-13T18:27:25,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445929 2024-11-13T18:27:25,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445929 2024-11-13T18:27:25,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445929) roll requested 2024-11-13T18:27:25,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445940 2024-11-13T18:27:25,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445940 2024-11-13T18:27:25,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445940) roll requested 2024-11-13T18:27:25,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445953 2024-11-13T18:27:25,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445953 2024-11-13T18:27:25,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445953) roll requested 2024-11-13T18:27:25,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445961 2024-11-13T18:27:25,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445961 2024-11-13T18:27:25,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445961) roll requested 2024-11-13T18:27:25,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445973 2024-11-13T18:27:25,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445973 2024-11-13T18:27:25,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445973) roll requested 2024-11-13T18:27:25,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445981 2024-11-13T18:27:25,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445981 2024-11-13T18:27:25,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445981) roll requested 2024-11-13T18:27:25,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445989 2024-11-13T18:27:25,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:25,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445989 2024-11-13T18:27:25,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:25,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:25,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445989) roll requested 2024-11-13T18:27:25,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522445996 2024-11-13T18:27:26,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445996 2024-11-13T18:27:26,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522445996) roll requested 2024-11-13T18:27:26,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446004 2024-11-13T18:27:26,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522445996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446004 2024-11-13T18:27:26,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446004) roll requested 2024-11-13T18:27:26,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446013 2024-11-13T18:27:26,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446013 2024-11-13T18:27:26,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446013) roll requested 2024-11-13T18:27:26,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446019 2024-11-13T18:27:26,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446019 2024-11-13T18:27:26,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446019) roll requested 2024-11-13T18:27:26,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446027 2024-11-13T18:27:26,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446027 2024-11-13T18:27:26,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446027) roll requested 2024-11-13T18:27:26,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446035 2024-11-13T18:27:26,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446035 2024-11-13T18:27:26,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446035) roll requested 2024-11-13T18:27:26,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446044 2024-11-13T18:27:26,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446044 2024-11-13T18:27:26,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446044) roll requested 2024-11-13T18:27:26,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446053 2024-11-13T18:27:26,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446053 2024-11-13T18:27:26,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446053) roll requested 2024-11-13T18:27:26,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446064 2024-11-13T18:27:26,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446064 2024-11-13T18:27:26,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446064) roll requested 2024-11-13T18:27:26,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446073 2024-11-13T18:27:26,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446073 2024-11-13T18:27:26,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446073) roll requested 2024-11-13T18:27:26,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446084 2024-11-13T18:27:26,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446084 2024-11-13T18:27:26,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446084) roll requested 2024-11-13T18:27:26,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446093 2024-11-13T18:27:26,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446093 2024-11-13T18:27:26,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446093) roll requested 2024-11-13T18:27:26,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446103 2024-11-13T18:27:26,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446103 2024-11-13T18:27:26,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446103) roll requested 2024-11-13T18:27:26,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446109 2024-11-13T18:27:26,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446109 2024-11-13T18:27:26,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446109) roll requested 2024-11-13T18:27:26,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446117 2024-11-13T18:27:26,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446117 2024-11-13T18:27:26,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446117) roll requested 2024-11-13T18:27:26,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446124 2024-11-13T18:27:26,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446124 2024-11-13T18:27:26,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446124) roll requested 2024-11-13T18:27:26,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446131 2024-11-13T18:27:26,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446131 2024-11-13T18:27:26,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446131) roll requested 2024-11-13T18:27:26,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446138 2024-11-13T18:27:26,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446138 2024-11-13T18:27:26,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446138) roll requested 2024-11-13T18:27:26,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446146 2024-11-13T18:27:26,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446146 2024-11-13T18:27:26,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446146) roll requested 2024-11-13T18:27:26,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446153 2024-11-13T18:27:26,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446153 2024-11-13T18:27:26,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446153) roll requested 2024-11-13T18:27:26,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446161 2024-11-13T18:27:26,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446161 2024-11-13T18:27:26,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446161) roll requested 2024-11-13T18:27:26,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446168 2024-11-13T18:27:26,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446168 2024-11-13T18:27:26,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446168) roll requested 2024-11-13T18:27:26,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446179 2024-11-13T18:27:26,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446179 2024-11-13T18:27:26,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446179) roll requested 2024-11-13T18:27:26,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446186 2024-11-13T18:27:26,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446186 2024-11-13T18:27:26,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446186) roll requested 2024-11-13T18:27:26,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446194 2024-11-13T18:27:26,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446194 2024-11-13T18:27:26,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446194) roll requested 2024-11-13T18:27:26,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446205 2024-11-13T18:27:26,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446205 2024-11-13T18:27:26,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446205) roll requested 2024-11-13T18:27:26,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446213 2024-11-13T18:27:26,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446213 2024-11-13T18:27:26,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446213) roll requested 2024-11-13T18:27:26,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446220 2024-11-13T18:27:26,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446220 2024-11-13T18:27:26,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446220) roll requested 2024-11-13T18:27:26,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446227 2024-11-13T18:27:26,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446227 2024-11-13T18:27:26,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446227) roll requested 2024-11-13T18:27:26,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446234 2024-11-13T18:27:26,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446234 2024-11-13T18:27:26,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446234) roll requested 2024-11-13T18:27:26,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446240 2024-11-13T18:27:26,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446240 2024-11-13T18:27:26,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446240) roll requested 2024-11-13T18:27:26,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446247 2024-11-13T18:27:26,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446247 2024-11-13T18:27:26,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446247) roll requested 2024-11-13T18:27:26,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446253 2024-11-13T18:27:26,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446253 2024-11-13T18:27:26,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446253) roll requested 2024-11-13T18:27:26,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446260 2024-11-13T18:27:26,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446260 2024-11-13T18:27:26,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446260) roll requested 2024-11-13T18:27:26,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446267 2024-11-13T18:27:26,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446267 2024-11-13T18:27:26,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446267) roll requested 2024-11-13T18:27:26,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446275 2024-11-13T18:27:26,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446275 2024-11-13T18:27:26,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446275) roll requested 2024-11-13T18:27:26,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446281 2024-11-13T18:27:26,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446281 2024-11-13T18:27:26,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446281) roll requested 2024-11-13T18:27:26,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446288 2024-11-13T18:27:26,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446288 2024-11-13T18:27:26,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446288) roll requested 2024-11-13T18:27:26,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446294 2024-11-13T18:27:26,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446294 2024-11-13T18:27:26,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446294) roll requested 2024-11-13T18:27:26,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446301 2024-11-13T18:27:26,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446301 2024-11-13T18:27:26,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446301) roll requested 2024-11-13T18:27:26,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446312 2024-11-13T18:27:26,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446312 2024-11-13T18:27:26,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446312) roll requested 2024-11-13T18:27:26,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446323 2024-11-13T18:27:26,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446323 2024-11-13T18:27:26,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446323) roll requested 2024-11-13T18:27:26,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446330 2024-11-13T18:27:26,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446330 2024-11-13T18:27:26,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:26,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446330) roll requested 2024-11-13T18:27:26,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446337 2024-11-13T18:27:26,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446337 2024-11-13T18:27:26,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446337) roll requested 2024-11-13T18:27:26,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446345 2024-11-13T18:27:26,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446345 2024-11-13T18:27:26,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446345) roll requested 2024-11-13T18:27:26,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446358 2024-11-13T18:27:26,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446358 2024-11-13T18:27:26,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446358) roll requested 2024-11-13T18:27:26,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446367 2024-11-13T18:27:26,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446367 2024-11-13T18:27:26,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446367) roll requested 2024-11-13T18:27:26,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446377 2024-11-13T18:27:26,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446377 2024-11-13T18:27:26,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446377) roll requested 2024-11-13T18:27:26,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446386 2024-11-13T18:27:26,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446386 2024-11-13T18:27:26,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446386) roll requested 2024-11-13T18:27:26,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446399 2024-11-13T18:27:26,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446399 2024-11-13T18:27:26,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446399) roll requested 2024-11-13T18:27:26,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446410 2024-11-13T18:27:26,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446410 2024-11-13T18:27:26,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446410) roll requested 2024-11-13T18:27:26,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446419 2024-11-13T18:27:26,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446419 2024-11-13T18:27:26,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446419) roll requested 2024-11-13T18:27:26,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446426 2024-11-13T18:27:26,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446426 2024-11-13T18:27:26,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446426) roll requested 2024-11-13T18:27:26,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446433 2024-11-13T18:27:26,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446433 2024-11-13T18:27:26,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446433) roll requested 2024-11-13T18:27:26,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446441 2024-11-13T18:27:26,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446441 2024-11-13T18:27:26,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446441) roll requested 2024-11-13T18:27:26,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446460 2024-11-13T18:27:26,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446460 2024-11-13T18:27:26,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446460) roll requested 2024-11-13T18:27:26,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446468 2024-11-13T18:27:26,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446468 2024-11-13T18:27:26,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446468) roll requested 2024-11-13T18:27:26,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446476 2024-11-13T18:27:26,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446476 2024-11-13T18:27:26,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446476) roll requested 2024-11-13T18:27:26,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446484 2024-11-13T18:27:26,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446484 2024-11-13T18:27:26,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446484) roll requested 2024-11-13T18:27:26,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446490 2024-11-13T18:27:26,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446490 2024-11-13T18:27:26,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446490) roll requested 2024-11-13T18:27:26,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446497 2024-11-13T18:27:26,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446497 2024-11-13T18:27:26,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446497) roll requested 2024-11-13T18:27:26,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446504 2024-11-13T18:27:26,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446504 2024-11-13T18:27:26,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446504) roll requested 2024-11-13T18:27:26,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446511 2024-11-13T18:27:26,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446511 2024-11-13T18:27:26,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446511) roll requested 2024-11-13T18:27:26,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446519 2024-11-13T18:27:26,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446519 2024-11-13T18:27:26,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446519) roll requested 2024-11-13T18:27:26,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446527 2024-11-13T18:27:26,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446527 2024-11-13T18:27:26,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446527) roll requested 2024-11-13T18:27:26,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446538 2024-11-13T18:27:26,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446538 2024-11-13T18:27:26,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446538) roll requested 2024-11-13T18:27:26,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446548 2024-11-13T18:27:26,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446548 2024-11-13T18:27:26,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446548) roll requested 2024-11-13T18:27:26,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446555 2024-11-13T18:27:26,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446555 2024-11-13T18:27:26,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446555) roll requested 2024-11-13T18:27:26,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446568 2024-11-13T18:27:26,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446568 2024-11-13T18:27:26,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446568) roll requested 2024-11-13T18:27:26,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446580 2024-11-13T18:27:26,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446580 2024-11-13T18:27:26,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446580) roll requested 2024-11-13T18:27:26,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446590 2024-11-13T18:27:26,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446590 2024-11-13T18:27:26,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446590) roll requested 2024-11-13T18:27:26,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446598 2024-11-13T18:27:26,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446598 2024-11-13T18:27:26,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446598) roll requested 2024-11-13T18:27:26,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446606 2024-11-13T18:27:26,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446606 2024-11-13T18:27:26,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446606) roll requested 2024-11-13T18:27:26,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446617 2024-11-13T18:27:26,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446617 2024-11-13T18:27:26,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446617) roll requested 2024-11-13T18:27:26,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446629 2024-11-13T18:27:26,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446629 2024-11-13T18:27:26,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446629) roll requested 2024-11-13T18:27:26,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446638 2024-11-13T18:27:26,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446638 2024-11-13T18:27:26,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446638) roll requested 2024-11-13T18:27:26,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446649 2024-11-13T18:27:26,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446649 2024-11-13T18:27:26,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446649) roll requested 2024-11-13T18:27:26,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446656 2024-11-13T18:27:26,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446656 2024-11-13T18:27:26,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446656) roll requested 2024-11-13T18:27:26,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446663 2024-11-13T18:27:26,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446663 2024-11-13T18:27:26,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446663) roll requested 2024-11-13T18:27:26,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446670 2024-11-13T18:27:26,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446670 2024-11-13T18:27:26,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446670) roll requested 2024-11-13T18:27:26,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446677 2024-11-13T18:27:26,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446677 2024-11-13T18:27:26,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446677) roll requested 2024-11-13T18:27:26,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446684 2024-11-13T18:27:26,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446684 2024-11-13T18:27:26,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446684) roll requested 2024-11-13T18:27:26,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446692 2024-11-13T18:27:26,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446692 2024-11-13T18:27:26,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446692) roll requested 2024-11-13T18:27:26,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446699 2024-11-13T18:27:26,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446699 2024-11-13T18:27:26,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446699) roll requested 2024-11-13T18:27:26,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446707 2024-11-13T18:27:26,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446707 2024-11-13T18:27:26,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446707) roll requested 2024-11-13T18:27:26,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446715 2024-11-13T18:27:26,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446715 2024-11-13T18:27:26,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446715) roll requested 2024-11-13T18:27:26,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446723 2024-11-13T18:27:26,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446723 2024-11-13T18:27:26,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446723) roll requested 2024-11-13T18:27:26,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446730 2024-11-13T18:27:26,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446730 2024-11-13T18:27:26,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446730) roll requested 2024-11-13T18:27:26,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446738 2024-11-13T18:27:26,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446738 2024-11-13T18:27:26,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446738) roll requested 2024-11-13T18:27:26,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446746 2024-11-13T18:27:26,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446746 2024-11-13T18:27:26,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446746) roll requested 2024-11-13T18:27:26,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446752 2024-11-13T18:27:26,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446752 2024-11-13T18:27:26,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446752) roll requested 2024-11-13T18:27:26,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446759 2024-11-13T18:27:26,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446759 2024-11-13T18:27:26,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446759) roll requested 2024-11-13T18:27:26,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446767 2024-11-13T18:27:26,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446767 2024-11-13T18:27:26,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446767) roll requested 2024-11-13T18:27:26,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446774 2024-11-13T18:27:26,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446774 2024-11-13T18:27:26,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446774) roll requested 2024-11-13T18:27:26,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446782 2024-11-13T18:27:26,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446782 2024-11-13T18:27:26,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446782) roll requested 2024-11-13T18:27:26,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446795 2024-11-13T18:27:26,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446795 2024-11-13T18:27:26,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446795) roll requested 2024-11-13T18:27:26,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446809 2024-11-13T18:27:26,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446809 2024-11-13T18:27:26,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446809) roll requested 2024-11-13T18:27:26,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446821 2024-11-13T18:27:26,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446821 2024-11-13T18:27:26,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446821) roll requested 2024-11-13T18:27:26,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446829 2024-11-13T18:27:26,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446829 2024-11-13T18:27:26,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446829) roll requested 2024-11-13T18:27:26,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446839 2024-11-13T18:27:26,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446839 2024-11-13T18:27:26,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446839) roll requested 2024-11-13T18:27:26,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446846 2024-11-13T18:27:26,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446846 2024-11-13T18:27:26,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446846) roll requested 2024-11-13T18:27:26,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446853 2024-11-13T18:27:26,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446853 2024-11-13T18:27:26,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446853) roll requested 2024-11-13T18:27:26,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446861 2024-11-13T18:27:26,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446861 2024-11-13T18:27:26,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446861) roll requested 2024-11-13T18:27:26,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446869 2024-11-13T18:27:26,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446869 2024-11-13T18:27:26,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446869) roll requested 2024-11-13T18:27:26,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446877 2024-11-13T18:27:26,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446877 2024-11-13T18:27:26,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446877) roll requested 2024-11-13T18:27:26,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446885 2024-11-13T18:27:26,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446885 2024-11-13T18:27:26,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446885) roll requested 2024-11-13T18:27:26,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446894 2024-11-13T18:27:26,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446894 2024-11-13T18:27:26,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446894) roll requested 2024-11-13T18:27:26,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446901 2024-11-13T18:27:26,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446901 2024-11-13T18:27:26,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446901) roll requested 2024-11-13T18:27:26,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446909 2024-11-13T18:27:26,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446909 2024-11-13T18:27:26,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446909) roll requested 2024-11-13T18:27:26,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446922 2024-11-13T18:27:26,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446922 2024-11-13T18:27:26,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446922) roll requested 2024-11-13T18:27:26,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446939 2024-11-13T18:27:26,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446939 2024-11-13T18:27:26,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446939) roll requested 2024-11-13T18:27:26,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446953 2024-11-13T18:27:26,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446953 2024-11-13T18:27:26,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:26,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446953) roll requested 2024-11-13T18:27:26,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522446978 2024-11-13T18:27:26,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:26,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446978 2024-11-13T18:27:26,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:26,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:26,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:26,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522446978) roll requested 2024-11-13T18:27:27,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447001 2024-11-13T18:27:27,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522446978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447001 2024-11-13T18:27:27,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447001) roll requested 2024-11-13T18:27:27,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447017 2024-11-13T18:27:27,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447017 2024-11-13T18:27:27,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447017) roll requested 2024-11-13T18:27:27,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447048 2024-11-13T18:27:27,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447048 2024-11-13T18:27:27,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447048) roll requested 2024-11-13T18:27:27,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447077 2024-11-13T18:27:27,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447077 2024-11-13T18:27:27,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447077) roll requested 2024-11-13T18:27:27,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447095 2024-11-13T18:27:27,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447095 2024-11-13T18:27:27,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447095) roll requested 2024-11-13T18:27:27,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447115 2024-11-13T18:27:27,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447115 2024-11-13T18:27:27,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447115) roll requested 2024-11-13T18:27:27,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447180 2024-11-13T18:27:27,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447180 2024-11-13T18:27:27,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447180) roll requested 2024-11-13T18:27:27,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447201 2024-11-13T18:27:27,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447201 2024-11-13T18:27:27,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447201) roll requested 2024-11-13T18:27:27,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447215 2024-11-13T18:27:27,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447215 2024-11-13T18:27:27,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447215) roll requested 2024-11-13T18:27:27,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447240 2024-11-13T18:27:27,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447240 2024-11-13T18:27:27,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447240) roll requested 2024-11-13T18:27:27,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447255 2024-11-13T18:27:27,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447255 2024-11-13T18:27:27,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447255) roll requested 2024-11-13T18:27:27,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447269 2024-11-13T18:27:27,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447269 2024-11-13T18:27:27,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447269) roll requested 2024-11-13T18:27:27,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447290 2024-11-13T18:27:27,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447290 2024-11-13T18:27:27,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447290) roll requested 2024-11-13T18:27:27,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447299 2024-11-13T18:27:27,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447299 2024-11-13T18:27:27,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447299) roll requested 2024-11-13T18:27:27,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447324 2024-11-13T18:27:27,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447324 2024-11-13T18:27:27,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447324) roll requested 2024-11-13T18:27:27,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447370 2024-11-13T18:27:27,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447370 2024-11-13T18:27:27,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447370) roll requested 2024-11-13T18:27:27,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447406 2024-11-13T18:27:27,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447406 2024-11-13T18:27:27,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447406) roll requested 2024-11-13T18:27:27,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447429 2024-11-13T18:27:27,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447429 2024-11-13T18:27:27,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447429) roll requested 2024-11-13T18:27:27,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447444 2024-11-13T18:27:27,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447444 2024-11-13T18:27:27,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447444) roll requested 2024-11-13T18:27:27,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447470 2024-11-13T18:27:27,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447470 2024-11-13T18:27:27,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447470) roll requested 2024-11-13T18:27:27,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447485 2024-11-13T18:27:27,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447485 2024-11-13T18:27:27,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447485) roll requested 2024-11-13T18:27:27,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447505 2024-11-13T18:27:27,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447505 2024-11-13T18:27:27,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447505) roll requested 2024-11-13T18:27:27,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447522 2024-11-13T18:27:27,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447522 2024-11-13T18:27:27,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447522) roll requested 2024-11-13T18:27:27,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447543 2024-11-13T18:27:27,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447543 2024-11-13T18:27:27,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447543) roll requested 2024-11-13T18:27:27,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447570 2024-11-13T18:27:27,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447570 2024-11-13T18:27:27,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447570) roll requested 2024-11-13T18:27:27,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447585 2024-11-13T18:27:27,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447585 2024-11-13T18:27:27,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447585) roll requested 2024-11-13T18:27:27,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447593 2024-11-13T18:27:27,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447593 2024-11-13T18:27:27,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447593) roll requested 2024-11-13T18:27:27,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447607 2024-11-13T18:27:27,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447607 2024-11-13T18:27:27,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447607) roll requested 2024-11-13T18:27:27,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447614 2024-11-13T18:27:27,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447614 2024-11-13T18:27:27,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447614) roll requested 2024-11-13T18:27:27,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447621 2024-11-13T18:27:27,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447621 2024-11-13T18:27:27,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447621) roll requested 2024-11-13T18:27:27,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447633 2024-11-13T18:27:27,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447633 2024-11-13T18:27:27,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447633) roll requested 2024-11-13T18:27:27,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447643 2024-11-13T18:27:27,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447643 2024-11-13T18:27:27,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447643) roll requested 2024-11-13T18:27:27,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447652 2024-11-13T18:27:27,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447652 2024-11-13T18:27:27,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447652) roll requested 2024-11-13T18:27:27,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447660 2024-11-13T18:27:27,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447660 2024-11-13T18:27:27,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447660) roll requested 2024-11-13T18:27:27,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447712 2024-11-13T18:27:27,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447712 2024-11-13T18:27:27,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447712) roll requested 2024-11-13T18:27:27,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447745 2024-11-13T18:27:27,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447745 2024-11-13T18:27:27,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447745) roll requested 2024-11-13T18:27:27,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447761 2024-11-13T18:27:27,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447761 2024-11-13T18:27:27,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447761) roll requested 2024-11-13T18:27:27,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447776 2024-11-13T18:27:27,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447776 2024-11-13T18:27:27,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447776) roll requested 2024-11-13T18:27:27,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447785 2024-11-13T18:27:27,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447785 2024-11-13T18:27:27,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447785) roll requested 2024-11-13T18:27:27,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447793 2024-11-13T18:27:27,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447793 2024-11-13T18:27:27,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447793) roll requested 2024-11-13T18:27:27,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447800 2024-11-13T18:27:27,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447800 2024-11-13T18:27:27,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447800) roll requested 2024-11-13T18:27:27,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447808 2024-11-13T18:27:27,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447808 2024-11-13T18:27:27,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447808) roll requested 2024-11-13T18:27:27,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447816 2024-11-13T18:27:27,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447816 2024-11-13T18:27:27,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447816) roll requested 2024-11-13T18:27:27,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447825 2024-11-13T18:27:27,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447825 2024-11-13T18:27:27,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447825) roll requested 2024-11-13T18:27:27,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447837 2024-11-13T18:27:27,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447837 2024-11-13T18:27:27,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447837) roll requested 2024-11-13T18:27:27,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447852 2024-11-13T18:27:27,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447852 2024-11-13T18:27:27,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447852) roll requested 2024-11-13T18:27:27,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447865 2024-11-13T18:27:27,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447865 2024-11-13T18:27:27,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447865) roll requested 2024-11-13T18:27:27,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447877 2024-11-13T18:27:27,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447877 2024-11-13T18:27:27,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447877) roll requested 2024-11-13T18:27:27,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447891 2024-11-13T18:27:27,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447891 2024-11-13T18:27:27,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447891) roll requested 2024-11-13T18:27:27,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447899 2024-11-13T18:27:27,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447899 2024-11-13T18:27:27,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447899) roll requested 2024-11-13T18:27:27,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447907 2024-11-13T18:27:27,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447907 2024-11-13T18:27:27,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447907) roll requested 2024-11-13T18:27:27,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447915 2024-11-13T18:27:27,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447915 2024-11-13T18:27:27,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447915) roll requested 2024-11-13T18:27:27,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447922 2024-11-13T18:27:27,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447922 2024-11-13T18:27:27,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447922) roll requested 2024-11-13T18:27:27,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447930 2024-11-13T18:27:27,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447930 2024-11-13T18:27:27,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447930) roll requested 2024-11-13T18:27:27,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447939 2024-11-13T18:27:27,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447939 2024-11-13T18:27:27,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447939) roll requested 2024-11-13T18:27:27,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447946 2024-11-13T18:27:27,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447946 2024-11-13T18:27:27,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447946) roll requested 2024-11-13T18:27:27,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447954 2024-11-13T18:27:27,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447954 2024-11-13T18:27:27,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447954) roll requested 2024-11-13T18:27:27,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447961 2024-11-13T18:27:27,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447961 2024-11-13T18:27:27,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447961) roll requested 2024-11-13T18:27:27,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447968 2024-11-13T18:27:27,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447968 2024-11-13T18:27:27,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447968) roll requested 2024-11-13T18:27:27,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447978 2024-11-13T18:27:27,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447978 2024-11-13T18:27:27,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447978) roll requested 2024-11-13T18:27:27,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447986 2024-11-13T18:27:27,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:27,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447986 2024-11-13T18:27:27,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:27,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:27,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:27,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:27,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447986) roll requested 2024-11-13T18:27:27,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522447993 2024-11-13T18:27:28,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447993 2024-11-13T18:27:28,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522447993) roll requested 2024-11-13T18:27:28,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448002 2024-11-13T18:27:28,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522447993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448002 2024-11-13T18:27:28,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448002) roll requested 2024-11-13T18:27:28,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448010 2024-11-13T18:27:28,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448010 2024-11-13T18:27:28,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448010) roll requested 2024-11-13T18:27:28,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448017 2024-11-13T18:27:28,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448017 2024-11-13T18:27:28,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448017) roll requested 2024-11-13T18:27:28,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448025 2024-11-13T18:27:28,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448025 2024-11-13T18:27:28,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448025) roll requested 2024-11-13T18:27:28,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448034 2024-11-13T18:27:28,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448034 2024-11-13T18:27:28,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448034) roll requested 2024-11-13T18:27:28,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448044 2024-11-13T18:27:28,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448044 2024-11-13T18:27:28,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448044) roll requested 2024-11-13T18:27:28,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448055 2024-11-13T18:27:28,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448055 2024-11-13T18:27:28,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448055) roll requested 2024-11-13T18:27:28,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448063 2024-11-13T18:27:28,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448063 2024-11-13T18:27:28,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448063) roll requested 2024-11-13T18:27:28,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448072 2024-11-13T18:27:28,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448072 2024-11-13T18:27:28,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448072) roll requested 2024-11-13T18:27:28,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448080 2024-11-13T18:27:28,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448080 2024-11-13T18:27:28,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448080) roll requested 2024-11-13T18:27:28,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448093 2024-11-13T18:27:28,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448093 2024-11-13T18:27:28,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448093) roll requested 2024-11-13T18:27:28,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448101 2024-11-13T18:27:28,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448101 2024-11-13T18:27:28,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448101) roll requested 2024-11-13T18:27:28,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448108 2024-11-13T18:27:28,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448108 2024-11-13T18:27:28,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448108) roll requested 2024-11-13T18:27:28,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448117 2024-11-13T18:27:28,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448117 2024-11-13T18:27:28,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448117) roll requested 2024-11-13T18:27:28,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448125 2024-11-13T18:27:28,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448125 2024-11-13T18:27:28,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448125) roll requested 2024-11-13T18:27:28,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448138 2024-11-13T18:27:28,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448138 2024-11-13T18:27:28,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448138) roll requested 2024-11-13T18:27:28,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448148 2024-11-13T18:27:28,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448148 2024-11-13T18:27:28,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448148) roll requested 2024-11-13T18:27:28,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448155 2024-11-13T18:27:28,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448155 2024-11-13T18:27:28,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448155) roll requested 2024-11-13T18:27:28,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448162 2024-11-13T18:27:28,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448162 2024-11-13T18:27:28,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448162) roll requested 2024-11-13T18:27:28,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448171 2024-11-13T18:27:28,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448171 2024-11-13T18:27:28,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448171) roll requested 2024-11-13T18:27:28,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448178 2024-11-13T18:27:28,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448178 2024-11-13T18:27:28,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448178) roll requested 2024-11-13T18:27:28,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448185 2024-11-13T18:27:28,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448185 2024-11-13T18:27:28,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448185) roll requested 2024-11-13T18:27:28,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448192 2024-11-13T18:27:28,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448192 2024-11-13T18:27:28,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448192) roll requested 2024-11-13T18:27:28,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448199 2024-11-13T18:27:28,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448199 2024-11-13T18:27:28,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,206 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,206 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448199) roll requested 2024-11-13T18:27:28,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448207 2024-11-13T18:27:28,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448207 2024-11-13T18:27:28,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448207) roll requested 2024-11-13T18:27:28,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448214 2024-11-13T18:27:28,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448214 2024-11-13T18:27:28,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448214) roll requested 2024-11-13T18:27:28,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448221 2024-11-13T18:27:28,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448221 2024-11-13T18:27:28,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448221) roll requested 2024-11-13T18:27:28,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448228 2024-11-13T18:27:28,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448228 2024-11-13T18:27:28,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448228) roll requested 2024-11-13T18:27:28,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448235 2024-11-13T18:27:28,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448235 2024-11-13T18:27:28,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448235) roll requested 2024-11-13T18:27:28,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448242 2024-11-13T18:27:28,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448242 2024-11-13T18:27:28,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448242) roll requested 2024-11-13T18:27:28,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448249 2024-11-13T18:27:28,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448249 2024-11-13T18:27:28,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448249) roll requested 2024-11-13T18:27:28,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448257 2024-11-13T18:27:28,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448257 2024-11-13T18:27:28,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448257) roll requested 2024-11-13T18:27:28,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448264 2024-11-13T18:27:28,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448264 2024-11-13T18:27:28,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448264) roll requested 2024-11-13T18:27:28,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448272 2024-11-13T18:27:28,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448272 2024-11-13T18:27:28,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448272) roll requested 2024-11-13T18:27:28,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448280 2024-11-13T18:27:28,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448280 2024-11-13T18:27:28,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448280) roll requested 2024-11-13T18:27:28,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448288 2024-11-13T18:27:28,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448288 2024-11-13T18:27:28,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448288) roll requested 2024-11-13T18:27:28,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448295 2024-11-13T18:27:28,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448295 2024-11-13T18:27:28,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448295) roll requested 2024-11-13T18:27:28,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448302 2024-11-13T18:27:28,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448302 2024-11-13T18:27:28,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448302) roll requested 2024-11-13T18:27:28,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448309 2024-11-13T18:27:28,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448309 2024-11-13T18:27:28,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448309) roll requested 2024-11-13T18:27:28,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448317 2024-11-13T18:27:28,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448317 2024-11-13T18:27:28,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448317) roll requested 2024-11-13T18:27:28,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448325 2024-11-13T18:27:28,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448325 2024-11-13T18:27:28,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448325) roll requested 2024-11-13T18:27:28,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448333 2024-11-13T18:27:28,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448333 2024-11-13T18:27:28,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448333) roll requested 2024-11-13T18:27:28,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448343 2024-11-13T18:27:28,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448343 2024-11-13T18:27:28,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448343) roll requested 2024-11-13T18:27:28,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448355 2024-11-13T18:27:28,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448355 2024-11-13T18:27:28,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448355) roll requested 2024-11-13T18:27:28,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448363 2024-11-13T18:27:28,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448363 2024-11-13T18:27:28,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448363) roll requested 2024-11-13T18:27:28,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448371 2024-11-13T18:27:28,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448371 2024-11-13T18:27:28,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448371) roll requested 2024-11-13T18:27:28,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448378 2024-11-13T18:27:28,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448378 2024-11-13T18:27:28,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448378) roll requested 2024-11-13T18:27:28,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448386 2024-11-13T18:27:28,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448386 2024-11-13T18:27:28,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448386) roll requested 2024-11-13T18:27:28,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448393 2024-11-13T18:27:28,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448393 2024-11-13T18:27:28,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448393) roll requested 2024-11-13T18:27:28,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448400 2024-11-13T18:27:28,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448400 2024-11-13T18:27:28,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448400) roll requested 2024-11-13T18:27:28,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448406 2024-11-13T18:27:28,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448406 2024-11-13T18:27:28,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448406) roll requested 2024-11-13T18:27:28,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448413 2024-11-13T18:27:28,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448413 2024-11-13T18:27:28,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448413) roll requested 2024-11-13T18:27:28,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448420 2024-11-13T18:27:28,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448420 2024-11-13T18:27:28,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448420) roll requested 2024-11-13T18:27:28,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448427 2024-11-13T18:27:28,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448427 2024-11-13T18:27:28,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448427) roll requested 2024-11-13T18:27:28,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448434 2024-11-13T18:27:28,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448434 2024-11-13T18:27:28,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448434) roll requested 2024-11-13T18:27:28,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448442 2024-11-13T18:27:28,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448442 2024-11-13T18:27:28,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448442) roll requested 2024-11-13T18:27:28,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448450 2024-11-13T18:27:28,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448450 2024-11-13T18:27:28,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448450) roll requested 2024-11-13T18:27:28,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448457 2024-11-13T18:27:28,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448457 2024-11-13T18:27:28,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448457) roll requested 2024-11-13T18:27:28,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448464 2024-11-13T18:27:28,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448464 2024-11-13T18:27:28,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448464) roll requested 2024-11-13T18:27:28,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448471 2024-11-13T18:27:28,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448471 2024-11-13T18:27:28,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448471) roll requested 2024-11-13T18:27:28,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448478 2024-11-13T18:27:28,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448478 2024-11-13T18:27:28,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448478) roll requested 2024-11-13T18:27:28,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448487 2024-11-13T18:27:28,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448487 2024-11-13T18:27:28,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448487) roll requested 2024-11-13T18:27:28,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448494 2024-11-13T18:27:28,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448494 2024-11-13T18:27:28,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,502 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448494) roll requested 2024-11-13T18:27:28,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448502 2024-11-13T18:27:28,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448502 2024-11-13T18:27:28,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448502) roll requested 2024-11-13T18:27:28,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448509 2024-11-13T18:27:28,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448509 2024-11-13T18:27:28,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448509) roll requested 2024-11-13T18:27:28,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448518 2024-11-13T18:27:28,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448518 2024-11-13T18:27:28,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448518) roll requested 2024-11-13T18:27:28,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448529 2024-11-13T18:27:28,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448529 2024-11-13T18:27:28,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448529) roll requested 2024-11-13T18:27:28,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448537 2024-11-13T18:27:28,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448537 2024-11-13T18:27:28,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,545 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448537) roll requested 2024-11-13T18:27:28,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448546 2024-11-13T18:27:28,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448546 2024-11-13T18:27:28,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448546) roll requested 2024-11-13T18:27:28,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448555 2024-11-13T18:27:28,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448555 2024-11-13T18:27:28,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448555) roll requested 2024-11-13T18:27:28,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448562 2024-11-13T18:27:28,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448562 2024-11-13T18:27:28,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448562) roll requested 2024-11-13T18:27:28,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448569 2024-11-13T18:27:28,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448569 2024-11-13T18:27:28,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448569) roll requested 2024-11-13T18:27:28,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448577 2024-11-13T18:27:28,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448577 2024-11-13T18:27:28,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448577) roll requested 2024-11-13T18:27:28,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448584 2024-11-13T18:27:28,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448584 2024-11-13T18:27:28,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448584) roll requested 2024-11-13T18:27:28,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448591 2024-11-13T18:27:28,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448591 2024-11-13T18:27:28,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448591) roll requested 2024-11-13T18:27:28,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448598 2024-11-13T18:27:28,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448598 2024-11-13T18:27:28,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448598) roll requested 2024-11-13T18:27:28,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448605 2024-11-13T18:27:28,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448605 2024-11-13T18:27:28,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,613 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448605) roll requested 2024-11-13T18:27:28,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448613 2024-11-13T18:27:28,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448613 2024-11-13T18:27:28,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448613) roll requested 2024-11-13T18:27:28,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448620 2024-11-13T18:27:28,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448620 2024-11-13T18:27:28,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448620) roll requested 2024-11-13T18:27:28,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448627 2024-11-13T18:27:28,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448627 2024-11-13T18:27:28,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448627) roll requested 2024-11-13T18:27:28,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448640 2024-11-13T18:27:28,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448640 2024-11-13T18:27:28,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448640) roll requested 2024-11-13T18:27:28,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448648 2024-11-13T18:27:28,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448648 2024-11-13T18:27:28,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448648) roll requested 2024-11-13T18:27:28,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448654 2024-11-13T18:27:28,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448654 2024-11-13T18:27:28,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448654) roll requested 2024-11-13T18:27:28,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448661 2024-11-13T18:27:28,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448661 2024-11-13T18:27:28,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448661) roll requested 2024-11-13T18:27:28,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448669 2024-11-13T18:27:28,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448669 2024-11-13T18:27:28,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448669) roll requested 2024-11-13T18:27:28,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448677 2024-11-13T18:27:28,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448677 2024-11-13T18:27:28,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448677) roll requested 2024-11-13T18:27:28,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448685 2024-11-13T18:27:28,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448685 2024-11-13T18:27:28,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448685) roll requested 2024-11-13T18:27:28,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448693 2024-11-13T18:27:28,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448693 2024-11-13T18:27:28,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448693) roll requested 2024-11-13T18:27:28,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448701 2024-11-13T18:27:28,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448701 2024-11-13T18:27:28,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448701) roll requested 2024-11-13T18:27:28,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448708 2024-11-13T18:27:28,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448708 2024-11-13T18:27:28,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448708) roll requested 2024-11-13T18:27:28,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448715 2024-11-13T18:27:28,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448715 2024-11-13T18:27:28,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448715) roll requested 2024-11-13T18:27:28,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448723 2024-11-13T18:27:28,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448723 2024-11-13T18:27:28,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448723) roll requested 2024-11-13T18:27:28,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448729 2024-11-13T18:27:28,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448729 2024-11-13T18:27:28,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448729) roll requested 2024-11-13T18:27:28,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448737 2024-11-13T18:27:28,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448737 2024-11-13T18:27:28,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448737) roll requested 2024-11-13T18:27:28,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448746 2024-11-13T18:27:28,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448746 2024-11-13T18:27:28,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448746) roll requested 2024-11-13T18:27:28,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448756 2024-11-13T18:27:28,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448756 2024-11-13T18:27:28,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448756) roll requested 2024-11-13T18:27:28,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448764 2024-11-13T18:27:28,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448764 2024-11-13T18:27:28,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448764) roll requested 2024-11-13T18:27:28,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448771 2024-11-13T18:27:28,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448771 2024-11-13T18:27:28,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448771) roll requested 2024-11-13T18:27:28,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448778 2024-11-13T18:27:28,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448778 2024-11-13T18:27:28,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448778) roll requested 2024-11-13T18:27:28,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448785 2024-11-13T18:27:28,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448785 2024-11-13T18:27:28,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448785) roll requested 2024-11-13T18:27:28,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448793 2024-11-13T18:27:28,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448793 2024-11-13T18:27:28,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448793) roll requested 2024-11-13T18:27:28,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448800 2024-11-13T18:27:28,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448800 2024-11-13T18:27:28,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448800) roll requested 2024-11-13T18:27:28,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448807 2024-11-13T18:27:28,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448807 2024-11-13T18:27:28,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448807) roll requested 2024-11-13T18:27:28,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448813 2024-11-13T18:27:28,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448813 2024-11-13T18:27:28,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448813) roll requested 2024-11-13T18:27:28,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448821 2024-11-13T18:27:28,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448821 2024-11-13T18:27:28,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448821) roll requested 2024-11-13T18:27:28,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448827 2024-11-13T18:27:28,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448827 2024-11-13T18:27:28,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448827) roll requested 2024-11-13T18:27:28,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448835 2024-11-13T18:27:28,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448835 2024-11-13T18:27:28,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448835) roll requested 2024-11-13T18:27:28,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448845 2024-11-13T18:27:28,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448845 2024-11-13T18:27:28,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448845) roll requested 2024-11-13T18:27:28,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448865 2024-11-13T18:27:28,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448865 2024-11-13T18:27:28,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448865) roll requested 2024-11-13T18:27:28,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448892 2024-11-13T18:27:28,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448892 2024-11-13T18:27:28,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448892) roll requested 2024-11-13T18:27:28,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448901 2024-11-13T18:27:28,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448901 2024-11-13T18:27:28,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448901) roll requested 2024-11-13T18:27:28,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448912 2024-11-13T18:27:28,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448912 2024-11-13T18:27:28,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448912) roll requested 2024-11-13T18:27:28,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448921 2024-11-13T18:27:28,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448921 2024-11-13T18:27:28,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448921) roll requested 2024-11-13T18:27:28,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448928 2024-11-13T18:27:28,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448928 2024-11-13T18:27:28,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448928) roll requested 2024-11-13T18:27:28,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448934 2024-11-13T18:27:28,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448934 2024-11-13T18:27:28,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448934) roll requested 2024-11-13T18:27:28,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448941 2024-11-13T18:27:28,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448941 2024-11-13T18:27:28,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448941) roll requested 2024-11-13T18:27:28,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448952 2024-11-13T18:27:28,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448952 2024-11-13T18:27:28,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448952) roll requested 2024-11-13T18:27:28,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448970 2024-11-13T18:27:28,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448970 2024-11-13T18:27:28,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448970) roll requested 2024-11-13T18:27:28,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448978 2024-11-13T18:27:28,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448978 2024-11-13T18:27:28,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448978) roll requested 2024-11-13T18:27:28,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448985 2024-11-13T18:27:28,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448985 2024-11-13T18:27:28,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448985) roll requested 2024-11-13T18:27:28,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448992 2024-11-13T18:27:28,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:28,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448992 2024-11-13T18:27:28,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:28,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:28,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:28,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:28,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448992) roll requested 2024-11-13T18:27:28,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522448999 2024-11-13T18:27:29,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448999 2024-11-13T18:27:29,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522448999) roll requested 2024-11-13T18:27:29,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449006 2024-11-13T18:27:29,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522448999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449006 2024-11-13T18:27:29,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449006) roll requested 2024-11-13T18:27:29,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449014 2024-11-13T18:27:29,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449014 2024-11-13T18:27:29,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449014) roll requested 2024-11-13T18:27:29,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449021 2024-11-13T18:27:29,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449021 2024-11-13T18:27:29,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449021) roll requested 2024-11-13T18:27:29,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449030 2024-11-13T18:27:29,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449030 2024-11-13T18:27:29,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449030) roll requested 2024-11-13T18:27:29,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449037 2024-11-13T18:27:29,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449037 2024-11-13T18:27:29,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449037) roll requested 2024-11-13T18:27:29,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449044 2024-11-13T18:27:29,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449044 2024-11-13T18:27:29,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449044) roll requested 2024-11-13T18:27:29,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449052 2024-11-13T18:27:29,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449052 2024-11-13T18:27:29,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449052) roll requested 2024-11-13T18:27:29,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449060 2024-11-13T18:27:29,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449060 2024-11-13T18:27:29,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449060) roll requested 2024-11-13T18:27:29,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449070 2024-11-13T18:27:29,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449070 2024-11-13T18:27:29,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449070) roll requested 2024-11-13T18:27:29,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449078 2024-11-13T18:27:29,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449078 2024-11-13T18:27:29,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449078) roll requested 2024-11-13T18:27:29,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449086 2024-11-13T18:27:29,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449086 2024-11-13T18:27:29,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449086) roll requested 2024-11-13T18:27:29,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449101 2024-11-13T18:27:29,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449101 2024-11-13T18:27:29,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449101) roll requested 2024-11-13T18:27:29,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449114 2024-11-13T18:27:29,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449114 2024-11-13T18:27:29,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449114) roll requested 2024-11-13T18:27:29,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449130 2024-11-13T18:27:29,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449130 2024-11-13T18:27:29,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449130) roll requested 2024-11-13T18:27:29,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449141 2024-11-13T18:27:29,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449141 2024-11-13T18:27:29,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449141) roll requested 2024-11-13T18:27:29,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449150 2024-11-13T18:27:29,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449150 2024-11-13T18:27:29,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449150) roll requested 2024-11-13T18:27:29,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449166 2024-11-13T18:27:29,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449166 2024-11-13T18:27:29,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449166) roll requested 2024-11-13T18:27:29,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449175 2024-11-13T18:27:29,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449175 2024-11-13T18:27:29,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449175) roll requested 2024-11-13T18:27:29,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449185 2024-11-13T18:27:29,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449185 2024-11-13T18:27:29,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449185) roll requested 2024-11-13T18:27:29,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449197 2024-11-13T18:27:29,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449197 2024-11-13T18:27:29,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449197) roll requested 2024-11-13T18:27:29,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449204 2024-11-13T18:27:29,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449204 2024-11-13T18:27:29,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449204) roll requested 2024-11-13T18:27:29,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449219 2024-11-13T18:27:29,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449219 2024-11-13T18:27:29,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449219) roll requested 2024-11-13T18:27:29,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449245 2024-11-13T18:27:29,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449245 2024-11-13T18:27:29,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449219 is not closed yet, will try archiving it next time 2024-11-13T18:27:29,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449245) roll requested 2024-11-13T18:27:29,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449258 2024-11-13T18:27:29,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449258 2024-11-13T18:27:29,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449258) roll requested 2024-11-13T18:27:29,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449269 2024-11-13T18:27:29,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449269 2024-11-13T18:27:29,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449269) roll requested 2024-11-13T18:27:29,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449277 2024-11-13T18:27:29,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449277 2024-11-13T18:27:29,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449277) roll requested 2024-11-13T18:27:29,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449285 2024-11-13T18:27:29,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449285 2024-11-13T18:27:29,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449285) roll requested 2024-11-13T18:27:29,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449301 2024-11-13T18:27:29,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449301 2024-11-13T18:27:29,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449301) roll requested 2024-11-13T18:27:29,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449309 2024-11-13T18:27:29,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449309 2024-11-13T18:27:29,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449309) roll requested 2024-11-13T18:27:29,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449318 2024-11-13T18:27:29,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449318 2024-11-13T18:27:29,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449318) roll requested 2024-11-13T18:27:29,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449326 2024-11-13T18:27:29,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449326 2024-11-13T18:27:29,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449326) roll requested 2024-11-13T18:27:29,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449335 2024-11-13T18:27:29,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449335 2024-11-13T18:27:29,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449335) roll requested 2024-11-13T18:27:29,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449344 2024-11-13T18:27:29,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449344 2024-11-13T18:27:29,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449344) roll requested 2024-11-13T18:27:29,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449350 2024-11-13T18:27:29,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449350 2024-11-13T18:27:29,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449350) roll requested 2024-11-13T18:27:29,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449360 2024-11-13T18:27:29,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449360 2024-11-13T18:27:29,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449360) roll requested 2024-11-13T18:27:29,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449369 2024-11-13T18:27:29,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449369 2024-11-13T18:27:29,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449369) roll requested 2024-11-13T18:27:29,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449381 2024-11-13T18:27:29,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449381 2024-11-13T18:27:29,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449381) roll requested 2024-11-13T18:27:29,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449390 2024-11-13T18:27:29,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449390 2024-11-13T18:27:29,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449390) roll requested 2024-11-13T18:27:29,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449402 2024-11-13T18:27:29,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449402 2024-11-13T18:27:29,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449402) roll requested 2024-11-13T18:27:29,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449421 2024-11-13T18:27:29,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449421 2024-11-13T18:27:29,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449421) roll requested 2024-11-13T18:27:29,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449432 2024-11-13T18:27:29,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449432 2024-11-13T18:27:29,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449432) roll requested 2024-11-13T18:27:29,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449440 2024-11-13T18:27:29,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449440 2024-11-13T18:27:29,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449440) roll requested 2024-11-13T18:27:29,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449454 2024-11-13T18:27:29,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449454 2024-11-13T18:27:29,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449454) roll requested 2024-11-13T18:27:29,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449466 2024-11-13T18:27:29,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449466 2024-11-13T18:27:29,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449466) roll requested 2024-11-13T18:27:29,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449477 2024-11-13T18:27:29,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449477 2024-11-13T18:27:29,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449477) roll requested 2024-11-13T18:27:29,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449496 2024-11-13T18:27:29,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449496 2024-11-13T18:27:29,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449496) roll requested 2024-11-13T18:27:29,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449505 2024-11-13T18:27:29,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449505 2024-11-13T18:27:29,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449505) roll requested 2024-11-13T18:27:29,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449521 2024-11-13T18:27:29,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449521 2024-11-13T18:27:29,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449521) roll requested 2024-11-13T18:27:29,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449532 2024-11-13T18:27:29,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449532 2024-11-13T18:27:29,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449532) roll requested 2024-11-13T18:27:29,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449553 2024-11-13T18:27:29,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449553 2024-11-13T18:27:29,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449553) roll requested 2024-11-13T18:27:29,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449565 2024-11-13T18:27:29,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449565 2024-11-13T18:27:29,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449565) roll requested 2024-11-13T18:27:29,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449575 2024-11-13T18:27:29,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449575 2024-11-13T18:27:29,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449575) roll requested 2024-11-13T18:27:29,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449582 2024-11-13T18:27:29,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449582 2024-11-13T18:27:29,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449582) roll requested 2024-11-13T18:27:29,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449591 2024-11-13T18:27:29,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449591 2024-11-13T18:27:29,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449591) roll requested 2024-11-13T18:27:29,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449600 2024-11-13T18:27:29,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449600 2024-11-13T18:27:29,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449600) roll requested 2024-11-13T18:27:29,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449608 2024-11-13T18:27:29,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449608 2024-11-13T18:27:29,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449608) roll requested 2024-11-13T18:27:29,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449616 2024-11-13T18:27:29,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449616 2024-11-13T18:27:29,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449616) roll requested 2024-11-13T18:27:29,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449625 2024-11-13T18:27:29,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449625 2024-11-13T18:27:29,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449625) roll requested 2024-11-13T18:27:29,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449633 2024-11-13T18:27:29,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449633 2024-11-13T18:27:29,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449633) roll requested 2024-11-13T18:27:29,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449641 2024-11-13T18:27:29,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449641 2024-11-13T18:27:29,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449641) roll requested 2024-11-13T18:27:29,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449650 2024-11-13T18:27:29,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449650 2024-11-13T18:27:29,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449650) roll requested 2024-11-13T18:27:29,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449657 2024-11-13T18:27:29,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449657 2024-11-13T18:27:29,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449657) roll requested 2024-11-13T18:27:29,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449680 2024-11-13T18:27:29,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449680 2024-11-13T18:27:29,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449680) roll requested 2024-11-13T18:27:29,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449693 2024-11-13T18:27:29,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449693 2024-11-13T18:27:29,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449693) roll requested 2024-11-13T18:27:29,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449701 2024-11-13T18:27:29,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449701 2024-11-13T18:27:29,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449701) roll requested 2024-11-13T18:27:29,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449709 2024-11-13T18:27:29,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449709 2024-11-13T18:27:29,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449709) roll requested 2024-11-13T18:27:29,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449717 2024-11-13T18:27:29,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449717 2024-11-13T18:27:29,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449717) roll requested 2024-11-13T18:27:29,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449724 2024-11-13T18:27:29,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449724 2024-11-13T18:27:29,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449724) roll requested 2024-11-13T18:27:29,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449731 2024-11-13T18:27:29,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449731 2024-11-13T18:27:29,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449731) roll requested 2024-11-13T18:27:29,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449739 2024-11-13T18:27:29,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449739 2024-11-13T18:27:29,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449739) roll requested 2024-11-13T18:27:29,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449746 2024-11-13T18:27:29,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449746 2024-11-13T18:27:29,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,754 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449746) roll requested 2024-11-13T18:27:29,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449754 2024-11-13T18:27:29,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449754 2024-11-13T18:27:29,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,762 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449754) roll requested 2024-11-13T18:27:29,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449762 2024-11-13T18:27:29,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449762 2024-11-13T18:27:29,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449762) roll requested 2024-11-13T18:27:29,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449771 2024-11-13T18:27:29,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449771 2024-11-13T18:27:29,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449771) roll requested 2024-11-13T18:27:29,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449779 2024-11-13T18:27:29,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449779 2024-11-13T18:27:29,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449779) roll requested 2024-11-13T18:27:29,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449791 2024-11-13T18:27:29,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449791 2024-11-13T18:27:29,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449791) roll requested 2024-11-13T18:27:29,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449799 2024-11-13T18:27:29,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449799 2024-11-13T18:27:29,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449799) roll requested 2024-11-13T18:27:29,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449806 2024-11-13T18:27:29,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449806 2024-11-13T18:27:29,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449806) roll requested 2024-11-13T18:27:29,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449814 2024-11-13T18:27:29,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449814 2024-11-13T18:27:29,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449814) roll requested 2024-11-13T18:27:29,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449822 2024-11-13T18:27:29,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449822 2024-11-13T18:27:29,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449822) roll requested 2024-11-13T18:27:29,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449830 2024-11-13T18:27:29,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449830 2024-11-13T18:27:29,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449830) roll requested 2024-11-13T18:27:29,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449839 2024-11-13T18:27:29,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449839 2024-11-13T18:27:29,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449839) roll requested 2024-11-13T18:27:29,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449849 2024-11-13T18:27:29,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449849 2024-11-13T18:27:29,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449849) roll requested 2024-11-13T18:27:29,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449856 2024-11-13T18:27:29,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449856 2024-11-13T18:27:29,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449856) roll requested 2024-11-13T18:27:29,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449864 2024-11-13T18:27:29,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449864 2024-11-13T18:27:29,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449864) roll requested 2024-11-13T18:27:29,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449873 2024-11-13T18:27:29,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449873 2024-11-13T18:27:29,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449873) roll requested 2024-11-13T18:27:29,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449882 2024-11-13T18:27:29,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449882 2024-11-13T18:27:29,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449882) roll requested 2024-11-13T18:27:29,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449890 2024-11-13T18:27:29,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449890 2024-11-13T18:27:29,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449890) roll requested 2024-11-13T18:27:29,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449899 2024-11-13T18:27:29,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449899 2024-11-13T18:27:29,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449899) roll requested 2024-11-13T18:27:29,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449907 2024-11-13T18:27:29,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449907 2024-11-13T18:27:29,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449907) roll requested 2024-11-13T18:27:29,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449915 2024-11-13T18:27:29,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449915 2024-11-13T18:27:29,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449915) roll requested 2024-11-13T18:27:29,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449926 2024-11-13T18:27:29,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449926 2024-11-13T18:27:29,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449926) roll requested 2024-11-13T18:27:29,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449934 2024-11-13T18:27:29,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449934 2024-11-13T18:27:29,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449934) roll requested 2024-11-13T18:27:29,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449940 2024-11-13T18:27:29,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449940 2024-11-13T18:27:29,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449940) roll requested 2024-11-13T18:27:29,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449949 2024-11-13T18:27:29,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449949 2024-11-13T18:27:29,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449949) roll requested 2024-11-13T18:27:29,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449957 2024-11-13T18:27:29,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449957 2024-11-13T18:27:29,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449957) roll requested 2024-11-13T18:27:29,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449965 2024-11-13T18:27:29,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449965 2024-11-13T18:27:29,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449965) roll requested 2024-11-13T18:27:29,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449973 2024-11-13T18:27:29,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449973 2024-11-13T18:27:29,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449973) roll requested 2024-11-13T18:27:29,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449984 2024-11-13T18:27:29,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449984 2024-11-13T18:27:29,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:29,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:29,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:29,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:29,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449984) roll requested 2024-11-13T18:27:29,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522449993 2024-11-13T18:27:29,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:29,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449993 2024-11-13T18:27:30,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522449993) roll requested 2024-11-13T18:27:30,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450001 2024-11-13T18:27:30,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522449993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450001 2024-11-13T18:27:30,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450001) roll requested 2024-11-13T18:27:30,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450025 2024-11-13T18:27:30,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450025 2024-11-13T18:27:30,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450025) roll requested 2024-11-13T18:27:30,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450034 2024-11-13T18:27:30,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450034 2024-11-13T18:27:30,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450034) roll requested 2024-11-13T18:27:30,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450043 2024-11-13T18:27:30,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450043 2024-11-13T18:27:30,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450043) roll requested 2024-11-13T18:27:30,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450052 2024-11-13T18:27:30,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450052 2024-11-13T18:27:30,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450052) roll requested 2024-11-13T18:27:30,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450075 2024-11-13T18:27:30,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450075 2024-11-13T18:27:30,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450075) roll requested 2024-11-13T18:27:30,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450106 2024-11-13T18:27:30,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450106 2024-11-13T18:27:30,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450106) roll requested 2024-11-13T18:27:30,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450129 2024-11-13T18:27:30,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450129 2024-11-13T18:27:30,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450129) roll requested 2024-11-13T18:27:30,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450162 2024-11-13T18:27:30,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450162 2024-11-13T18:27:30,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450162) roll requested 2024-11-13T18:27:30,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450209 2024-11-13T18:27:30,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450209 2024-11-13T18:27:30,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450209) roll requested 2024-11-13T18:27:30,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450223 2024-11-13T18:27:30,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450223 2024-11-13T18:27:30,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450223) roll requested 2024-11-13T18:27:30,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450233 2024-11-13T18:27:30,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450233 2024-11-13T18:27:30,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450233) roll requested 2024-11-13T18:27:30,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450247 2024-11-13T18:27:30,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450247 2024-11-13T18:27:30,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450247) roll requested 2024-11-13T18:27:30,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450254 2024-11-13T18:27:30,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450254 2024-11-13T18:27:30,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450254) roll requested 2024-11-13T18:27:30,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450266 2024-11-13T18:27:30,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450266 2024-11-13T18:27:30,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450266) roll requested 2024-11-13T18:27:30,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450277 2024-11-13T18:27:30,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450277 2024-11-13T18:27:30,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450277) roll requested 2024-11-13T18:27:30,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450301 2024-11-13T18:27:30,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450301 2024-11-13T18:27:30,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450301) roll requested 2024-11-13T18:27:30,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450313 2024-11-13T18:27:30,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450313 2024-11-13T18:27:30,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450313) roll requested 2024-11-13T18:27:30,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450329 2024-11-13T18:27:30,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450329 2024-11-13T18:27:30,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450329) roll requested 2024-11-13T18:27:30,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450339 2024-11-13T18:27:30,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450339 2024-11-13T18:27:30,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450339) roll requested 2024-11-13T18:27:30,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450349 2024-11-13T18:27:30,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450349 2024-11-13T18:27:30,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450349) roll requested 2024-11-13T18:27:30,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450357 2024-11-13T18:27:30,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450357 2024-11-13T18:27:30,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450357) roll requested 2024-11-13T18:27:30,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450364 2024-11-13T18:27:30,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450364 2024-11-13T18:27:30,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450364) roll requested 2024-11-13T18:27:30,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450373 2024-11-13T18:27:30,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450373 2024-11-13T18:27:30,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450373) roll requested 2024-11-13T18:27:30,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450389 2024-11-13T18:27:30,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450389 2024-11-13T18:27:30,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450389) roll requested 2024-11-13T18:27:30,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450406 2024-11-13T18:27:30,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450406 2024-11-13T18:27:30,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450406) roll requested 2024-11-13T18:27:30,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450422 2024-11-13T18:27:30,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450422 2024-11-13T18:27:30,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450422) roll requested 2024-11-13T18:27:30,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450437 2024-11-13T18:27:30,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450437 2024-11-13T18:27:30,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450437) roll requested 2024-11-13T18:27:30,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450454 2024-11-13T18:27:30,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450454 2024-11-13T18:27:30,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450454) roll requested 2024-11-13T18:27:30,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450465 2024-11-13T18:27:30,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450465 2024-11-13T18:27:30,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450465) roll requested 2024-11-13T18:27:30,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450478 2024-11-13T18:27:30,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450478 2024-11-13T18:27:30,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450478) roll requested 2024-11-13T18:27:30,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450485 2024-11-13T18:27:30,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450485 2024-11-13T18:27:30,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450485) roll requested 2024-11-13T18:27:30,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450492 2024-11-13T18:27:30,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450492 2024-11-13T18:27:30,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450492) roll requested 2024-11-13T18:27:30,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450500 2024-11-13T18:27:30,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450500 2024-11-13T18:27:30,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450500) roll requested 2024-11-13T18:27:30,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450508 2024-11-13T18:27:30,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450508 2024-11-13T18:27:30,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450508) roll requested 2024-11-13T18:27:30,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450516 2024-11-13T18:27:30,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450516 2024-11-13T18:27:30,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450516) roll requested 2024-11-13T18:27:30,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450523 2024-11-13T18:27:30,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450523 2024-11-13T18:27:30,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450523) roll requested 2024-11-13T18:27:30,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450531 2024-11-13T18:27:30,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450531 2024-11-13T18:27:30,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450531) roll requested 2024-11-13T18:27:30,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450543 2024-11-13T18:27:30,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450543 2024-11-13T18:27:30,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450543) roll requested 2024-11-13T18:27:30,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450554 2024-11-13T18:27:30,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450554 2024-11-13T18:27:30,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450554) roll requested 2024-11-13T18:27:30,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450563 2024-11-13T18:27:30,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450563 2024-11-13T18:27:30,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450563) roll requested 2024-11-13T18:27:30,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450571 2024-11-13T18:27:30,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450571 2024-11-13T18:27:30,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450571) roll requested 2024-11-13T18:27:30,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450579 2024-11-13T18:27:30,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450579 2024-11-13T18:27:30,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450579) roll requested 2024-11-13T18:27:30,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450586 2024-11-13T18:27:30,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450586 2024-11-13T18:27:30,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450586) roll requested 2024-11-13T18:27:30,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450597 2024-11-13T18:27:30,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450597 2024-11-13T18:27:30,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450597) roll requested 2024-11-13T18:27:30,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450607 2024-11-13T18:27:30,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450607 2024-11-13T18:27:30,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450607) roll requested 2024-11-13T18:27:30,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450621 2024-11-13T18:27:30,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450621 2024-11-13T18:27:30,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450621) roll requested 2024-11-13T18:27:30,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450633 2024-11-13T18:27:30,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450633 2024-11-13T18:27:30,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450633) roll requested 2024-11-13T18:27:30,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450645 2024-11-13T18:27:30,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450645 2024-11-13T18:27:30,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450645) roll requested 2024-11-13T18:27:30,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450652 2024-11-13T18:27:30,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450652 2024-11-13T18:27:30,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450652) roll requested 2024-11-13T18:27:30,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450659 2024-11-13T18:27:30,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450659 2024-11-13T18:27:30,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450659) roll requested 2024-11-13T18:27:30,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450667 2024-11-13T18:27:30,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450667 2024-11-13T18:27:30,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450667) roll requested 2024-11-13T18:27:30,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450675 2024-11-13T18:27:30,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450675 2024-11-13T18:27:30,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450675) roll requested 2024-11-13T18:27:30,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450682 2024-11-13T18:27:30,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450682 2024-11-13T18:27:30,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450682) roll requested 2024-11-13T18:27:30,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450689 2024-11-13T18:27:30,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450689 2024-11-13T18:27:30,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450689) roll requested 2024-11-13T18:27:30,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450696 2024-11-13T18:27:30,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450696 2024-11-13T18:27:30,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450696) roll requested 2024-11-13T18:27:30,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450704 2024-11-13T18:27:30,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450704 2024-11-13T18:27:30,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450704) roll requested 2024-11-13T18:27:30,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450721 2024-11-13T18:27:30,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450721 2024-11-13T18:27:30,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450721) roll requested 2024-11-13T18:27:30,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450730 2024-11-13T18:27:30,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450730 2024-11-13T18:27:30,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450730) roll requested 2024-11-13T18:27:30,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450738 2024-11-13T18:27:30,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450738 2024-11-13T18:27:30,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450738) roll requested 2024-11-13T18:27:30,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450749 2024-11-13T18:27:30,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450749 2024-11-13T18:27:30,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450749) roll requested 2024-11-13T18:27:30,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450757 2024-11-13T18:27:30,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450757 2024-11-13T18:27:30,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450757) roll requested 2024-11-13T18:27:30,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450765 2024-11-13T18:27:30,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450765 2024-11-13T18:27:30,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450765) roll requested 2024-11-13T18:27:30,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450778 2024-11-13T18:27:30,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450778 2024-11-13T18:27:30,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450778) roll requested 2024-11-13T18:27:30,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450785 2024-11-13T18:27:30,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450785 2024-11-13T18:27:30,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450785) roll requested 2024-11-13T18:27:30,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450792 2024-11-13T18:27:30,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450792 2024-11-13T18:27:30,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450792) roll requested 2024-11-13T18:27:30,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450801 2024-11-13T18:27:30,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450801 2024-11-13T18:27:30,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450801) roll requested 2024-11-13T18:27:30,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450808 2024-11-13T18:27:30,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450808 2024-11-13T18:27:30,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450808) roll requested 2024-11-13T18:27:30,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450815 2024-11-13T18:27:30,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450815 2024-11-13T18:27:30,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450815) roll requested 2024-11-13T18:27:30,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450822 2024-11-13T18:27:30,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450822 2024-11-13T18:27:30,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450822) roll requested 2024-11-13T18:27:30,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450828 2024-11-13T18:27:30,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450828 2024-11-13T18:27:30,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450828) roll requested 2024-11-13T18:27:30,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450835 2024-11-13T18:27:30,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450835 2024-11-13T18:27:30,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450835) roll requested 2024-11-13T18:27:30,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450842 2024-11-13T18:27:30,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450842 2024-11-13T18:27:30,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450842) roll requested 2024-11-13T18:27:30,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450849 2024-11-13T18:27:30,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450849 2024-11-13T18:27:30,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450849) roll requested 2024-11-13T18:27:30,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450856 2024-11-13T18:27:30,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450856 2024-11-13T18:27:30,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450856) roll requested 2024-11-13T18:27:30,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450865 2024-11-13T18:27:30,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450865 2024-11-13T18:27:30,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450865) roll requested 2024-11-13T18:27:30,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450871 2024-11-13T18:27:30,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450871 2024-11-13T18:27:30,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450871) roll requested 2024-11-13T18:27:30,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450879 2024-11-13T18:27:30,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450879 2024-11-13T18:27:30,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450879) roll requested 2024-11-13T18:27:30,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450889 2024-11-13T18:27:30,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450889 2024-11-13T18:27:30,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450889) roll requested 2024-11-13T18:27:30,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450896 2024-11-13T18:27:30,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450896 2024-11-13T18:27:30,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450896) roll requested 2024-11-13T18:27:30,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450903 2024-11-13T18:27:30,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450903 2024-11-13T18:27:30,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450903) roll requested 2024-11-13T18:27:30,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450915 2024-11-13T18:27:30,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450915 2024-11-13T18:27:30,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450915) roll requested 2024-11-13T18:27:30,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450925 2024-11-13T18:27:30,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450925 2024-11-13T18:27:30,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450925) roll requested 2024-11-13T18:27:30,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450931 2024-11-13T18:27:30,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450931 2024-11-13T18:27:30,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450931) roll requested 2024-11-13T18:27:30,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450938 2024-11-13T18:27:30,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450938 2024-11-13T18:27:30,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450938) roll requested 2024-11-13T18:27:30,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450950 2024-11-13T18:27:30,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450950 2024-11-13T18:27:30,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450950) roll requested 2024-11-13T18:27:30,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450959 2024-11-13T18:27:30,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450959 2024-11-13T18:27:30,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450959) roll requested 2024-11-13T18:27:30,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450966 2024-11-13T18:27:30,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450966 2024-11-13T18:27:30,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450966) roll requested 2024-11-13T18:27:30,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450974 2024-11-13T18:27:30,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450974 2024-11-13T18:27:30,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450974) roll requested 2024-11-13T18:27:30,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450982 2024-11-13T18:27:30,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:30,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450982 2024-11-13T18:27:30,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:30,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:30,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:30,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:30,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450982) roll requested 2024-11-13T18:27:30,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522450995 2024-11-13T18:27:31,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450995 2024-11-13T18:27:31,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522450995) roll requested 2024-11-13T18:27:31,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451004 2024-11-13T18:27:31,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522450995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451004 2024-11-13T18:27:31,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451004) roll requested 2024-11-13T18:27:31,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451012 2024-11-13T18:27:31,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451012 2024-11-13T18:27:31,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451012) roll requested 2024-11-13T18:27:31,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451024 2024-11-13T18:27:31,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451024 2024-11-13T18:27:31,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451024) roll requested 2024-11-13T18:27:31,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451032 2024-11-13T18:27:31,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451032 2024-11-13T18:27:31,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451032) roll requested 2024-11-13T18:27:31,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451040 2024-11-13T18:27:31,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451040 2024-11-13T18:27:31,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451040) roll requested 2024-11-13T18:27:31,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451048 2024-11-13T18:27:31,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451048 2024-11-13T18:27:31,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451048) roll requested 2024-11-13T18:27:31,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451060 2024-11-13T18:27:31,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451060 2024-11-13T18:27:31,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451060) roll requested 2024-11-13T18:27:31,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451083 2024-11-13T18:27:31,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451083 2024-11-13T18:27:31,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451083) roll requested 2024-11-13T18:27:31,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451101 2024-11-13T18:27:31,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451101 2024-11-13T18:27:31,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451101) roll requested 2024-11-13T18:27:31,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451111 2024-11-13T18:27:31,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451111 2024-11-13T18:27:31,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451111) roll requested 2024-11-13T18:27:31,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451136 2024-11-13T18:27:31,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451136 2024-11-13T18:27:31,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451136) roll requested 2024-11-13T18:27:31,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451149 2024-11-13T18:27:31,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451149 2024-11-13T18:27:31,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451149) roll requested 2024-11-13T18:27:31,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451169 2024-11-13T18:27:31,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451169 2024-11-13T18:27:31,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451169) roll requested 2024-11-13T18:27:31,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451185 2024-11-13T18:27:31,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451185 2024-11-13T18:27:31,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451185) roll requested 2024-11-13T18:27:31,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451193 2024-11-13T18:27:31,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451193 2024-11-13T18:27:31,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451193) roll requested 2024-11-13T18:27:31,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451205 2024-11-13T18:27:31,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451205 2024-11-13T18:27:31,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451205) roll requested 2024-11-13T18:27:31,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451217 2024-11-13T18:27:31,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451217 2024-11-13T18:27:31,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451217) roll requested 2024-11-13T18:27:31,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451229 2024-11-13T18:27:31,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451229 2024-11-13T18:27:31,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451229) roll requested 2024-11-13T18:27:31,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451237 2024-11-13T18:27:31,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451237 2024-11-13T18:27:31,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451237) roll requested 2024-11-13T18:27:31,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451245 2024-11-13T18:27:31,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451245 2024-11-13T18:27:31,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451245) roll requested 2024-11-13T18:27:31,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451256 2024-11-13T18:27:31,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451256 2024-11-13T18:27:31,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451256) roll requested 2024-11-13T18:27:31,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451264 2024-11-13T18:27:31,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451264 2024-11-13T18:27:31,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451264) roll requested 2024-11-13T18:27:31,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451270 2024-11-13T18:27:31,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451270 2024-11-13T18:27:31,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451270) roll requested 2024-11-13T18:27:31,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451277 2024-11-13T18:27:31,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451277 2024-11-13T18:27:31,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451277) roll requested 2024-11-13T18:27:31,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451285 2024-11-13T18:27:31,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451285 2024-11-13T18:27:31,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451285) roll requested 2024-11-13T18:27:31,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451293 2024-11-13T18:27:31,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451293 2024-11-13T18:27:31,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451293) roll requested 2024-11-13T18:27:31,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451301 2024-11-13T18:27:31,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451301 2024-11-13T18:27:31,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451301) roll requested 2024-11-13T18:27:31,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451308 2024-11-13T18:27:31,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451308 2024-11-13T18:27:31,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451308) roll requested 2024-11-13T18:27:31,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451321 2024-11-13T18:27:31,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451321 2024-11-13T18:27:31,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451321) roll requested 2024-11-13T18:27:31,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451333 2024-11-13T18:27:31,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451333 2024-11-13T18:27:31,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451333) roll requested 2024-11-13T18:27:31,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451342 2024-11-13T18:27:31,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451342 2024-11-13T18:27:31,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451342) roll requested 2024-11-13T18:27:31,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451351 2024-11-13T18:27:31,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451351 2024-11-13T18:27:31,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451351) roll requested 2024-11-13T18:27:31,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451362 2024-11-13T18:27:31,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451362 2024-11-13T18:27:31,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451362) roll requested 2024-11-13T18:27:31,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451371 2024-11-13T18:27:31,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451371 2024-11-13T18:27:31,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451371) roll requested 2024-11-13T18:27:31,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451380 2024-11-13T18:27:31,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451380 2024-11-13T18:27:31,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451380) roll requested 2024-11-13T18:27:31,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451391 2024-11-13T18:27:31,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451391 2024-11-13T18:27:31,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451391) roll requested 2024-11-13T18:27:31,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451401 2024-11-13T18:27:31,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451401 2024-11-13T18:27:31,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451401) roll requested 2024-11-13T18:27:31,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451408 2024-11-13T18:27:31,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451408 2024-11-13T18:27:31,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451408) roll requested 2024-11-13T18:27:31,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451417 2024-11-13T18:27:31,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451417 2024-11-13T18:27:31,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451417) roll requested 2024-11-13T18:27:31,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451425 2024-11-13T18:27:31,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451425 2024-11-13T18:27:31,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451425) roll requested 2024-11-13T18:27:31,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451432 2024-11-13T18:27:31,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451432 2024-11-13T18:27:31,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451432) roll requested 2024-11-13T18:27:31,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451442 2024-11-13T18:27:31,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451442 2024-11-13T18:27:31,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451442) roll requested 2024-11-13T18:27:31,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451449 2024-11-13T18:27:31,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451449 2024-11-13T18:27:31,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451449) roll requested 2024-11-13T18:27:31,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451460 2024-11-13T18:27:31,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451460 2024-11-13T18:27:31,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451460) roll requested 2024-11-13T18:27:31,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451470 2024-11-13T18:27:31,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451470 2024-11-13T18:27:31,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451470) roll requested 2024-11-13T18:27:31,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451478 2024-11-13T18:27:31,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451478 2024-11-13T18:27:31,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451478) roll requested 2024-11-13T18:27:31,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451485 2024-11-13T18:27:31,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451485 2024-11-13T18:27:31,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451485) roll requested 2024-11-13T18:27:31,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451492 2024-11-13T18:27:31,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451492 2024-11-13T18:27:31,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451492) roll requested 2024-11-13T18:27:31,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451498 2024-11-13T18:27:31,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451498 2024-11-13T18:27:31,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451498) roll requested 2024-11-13T18:27:31,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451505 2024-11-13T18:27:31,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451505 2024-11-13T18:27:31,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451505) roll requested 2024-11-13T18:27:31,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451511 2024-11-13T18:27:31,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451511 2024-11-13T18:27:31,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451511) roll requested 2024-11-13T18:27:31,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451518 2024-11-13T18:27:31,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451518 2024-11-13T18:27:31,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451518) roll requested 2024-11-13T18:27:31,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451525 2024-11-13T18:27:31,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451525 2024-11-13T18:27:31,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451525) roll requested 2024-11-13T18:27:31,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451532 2024-11-13T18:27:31,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451532 2024-11-13T18:27:31,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451532) roll requested 2024-11-13T18:27:31,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451539 2024-11-13T18:27:31,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451539 2024-11-13T18:27:31,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451539) roll requested 2024-11-13T18:27:31,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451546 2024-11-13T18:27:31,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451546 2024-11-13T18:27:31,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451546) roll requested 2024-11-13T18:27:31,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451553 2024-11-13T18:27:31,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451553 2024-11-13T18:27:31,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451553) roll requested 2024-11-13T18:27:31,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451566 2024-11-13T18:27:31,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451566 2024-11-13T18:27:31,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451566) roll requested 2024-11-13T18:27:31,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451574 2024-11-13T18:27:31,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451574 2024-11-13T18:27:31,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451574) roll requested 2024-11-13T18:27:31,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451582 2024-11-13T18:27:31,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451582 2024-11-13T18:27:31,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451582) roll requested 2024-11-13T18:27:31,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451589 2024-11-13T18:27:31,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451589 2024-11-13T18:27:31,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451589) roll requested 2024-11-13T18:27:31,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451597 2024-11-13T18:27:31,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451597 2024-11-13T18:27:31,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451597) roll requested 2024-11-13T18:27:31,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451605 2024-11-13T18:27:31,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451605 2024-11-13T18:27:31,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451605) roll requested 2024-11-13T18:27:31,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451616 2024-11-13T18:27:31,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451616 2024-11-13T18:27:31,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451616) roll requested 2024-11-13T18:27:31,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451624 2024-11-13T18:27:31,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451624 2024-11-13T18:27:31,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451624) roll requested 2024-11-13T18:27:31,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451633 2024-11-13T18:27:31,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451633 2024-11-13T18:27:31,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451633) roll requested 2024-11-13T18:27:31,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451644 2024-11-13T18:27:31,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451644 2024-11-13T18:27:31,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451644) roll requested 2024-11-13T18:27:31,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451653 2024-11-13T18:27:31,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451653 2024-11-13T18:27:31,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451653) roll requested 2024-11-13T18:27:31,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451664 2024-11-13T18:27:31,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451664 2024-11-13T18:27:31,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451664) roll requested 2024-11-13T18:27:31,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451673 2024-11-13T18:27:31,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451673 2024-11-13T18:27:31,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451673) roll requested 2024-11-13T18:27:31,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451680 2024-11-13T18:27:31,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451680 2024-11-13T18:27:31,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451680) roll requested 2024-11-13T18:27:31,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451687 2024-11-13T18:27:31,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451687 2024-11-13T18:27:31,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451687) roll requested 2024-11-13T18:27:31,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451695 2024-11-13T18:27:31,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451695 2024-11-13T18:27:31,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451695) roll requested 2024-11-13T18:27:31,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451708 2024-11-13T18:27:31,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451708 2024-11-13T18:27:31,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451708) roll requested 2024-11-13T18:27:31,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451723 2024-11-13T18:27:31,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451723 2024-11-13T18:27:31,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451723) roll requested 2024-11-13T18:27:31,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451737 2024-11-13T18:27:31,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451737 2024-11-13T18:27:31,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451737) roll requested 2024-11-13T18:27:31,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451749 2024-11-13T18:27:31,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451749 2024-11-13T18:27:31,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451749) roll requested 2024-11-13T18:27:31,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451756 2024-11-13T18:27:31,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451756 2024-11-13T18:27:31,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451756) roll requested 2024-11-13T18:27:31,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451767 2024-11-13T18:27:31,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451767 2024-11-13T18:27:31,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451767) roll requested 2024-11-13T18:27:31,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451775 2024-11-13T18:27:31,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451775 2024-11-13T18:27:31,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451775) roll requested 2024-11-13T18:27:31,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451784 2024-11-13T18:27:31,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451784 2024-11-13T18:27:31,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451784) roll requested 2024-11-13T18:27:31,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451793 2024-11-13T18:27:31,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451793 2024-11-13T18:27:31,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451793) roll requested 2024-11-13T18:27:31,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451801 2024-11-13T18:27:31,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451801 2024-11-13T18:27:31,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451801) roll requested 2024-11-13T18:27:31,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451808 2024-11-13T18:27:31,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451808 2024-11-13T18:27:31,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451808) roll requested 2024-11-13T18:27:31,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451816 2024-11-13T18:27:31,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451816 2024-11-13T18:27:31,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451816) roll requested 2024-11-13T18:27:31,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451823 2024-11-13T18:27:31,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451823 2024-11-13T18:27:31,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451823) roll requested 2024-11-13T18:27:31,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451835 2024-11-13T18:27:31,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451835 2024-11-13T18:27:31,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451835) roll requested 2024-11-13T18:27:31,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451843 2024-11-13T18:27:31,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451843 2024-11-13T18:27:31,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451843) roll requested 2024-11-13T18:27:31,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451852 2024-11-13T18:27:31,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451852 2024-11-13T18:27:31,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451852) roll requested 2024-11-13T18:27:31,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451860 2024-11-13T18:27:31,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451860 2024-11-13T18:27:31,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451860) roll requested 2024-11-13T18:27:31,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451870 2024-11-13T18:27:31,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451870 2024-11-13T18:27:31,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451870) roll requested 2024-11-13T18:27:31,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451878 2024-11-13T18:27:31,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451878 2024-11-13T18:27:31,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451878) roll requested 2024-11-13T18:27:31,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451889 2024-11-13T18:27:31,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451889 2024-11-13T18:27:31,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451889) roll requested 2024-11-13T18:27:31,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451897 2024-11-13T18:27:31,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451897 2024-11-13T18:27:31,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451897) roll requested 2024-11-13T18:27:31,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451907 2024-11-13T18:27:31,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451907 2024-11-13T18:27:31,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451907) roll requested 2024-11-13T18:27:31,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451915 2024-11-13T18:27:31,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451915 2024-11-13T18:27:31,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451915) roll requested 2024-11-13T18:27:31,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451922 2024-11-13T18:27:31,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451922 2024-11-13T18:27:31,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451922) roll requested 2024-11-13T18:27:31,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451928 2024-11-13T18:27:31,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451928 2024-11-13T18:27:31,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451928) roll requested 2024-11-13T18:27:31,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451936 2024-11-13T18:27:31,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451936 2024-11-13T18:27:31,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451936) roll requested 2024-11-13T18:27:31,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451944 2024-11-13T18:27:31,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451944 2024-11-13T18:27:31,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451944) roll requested 2024-11-13T18:27:31,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451951 2024-11-13T18:27:31,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451951 2024-11-13T18:27:31,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451951) roll requested 2024-11-13T18:27:31,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451959 2024-11-13T18:27:31,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451959 2024-11-13T18:27:31,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451959) roll requested 2024-11-13T18:27:31,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451966 2024-11-13T18:27:31,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451966 2024-11-13T18:27:31,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451966) roll requested 2024-11-13T18:27:31,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451974 2024-11-13T18:27:31,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451974 2024-11-13T18:27:31,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451974) roll requested 2024-11-13T18:27:31,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451983 2024-11-13T18:27:31,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451983 2024-11-13T18:27:31,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451983) roll requested 2024-11-13T18:27:31,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451991 2024-11-13T18:27:31,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:31,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451991 2024-11-13T18:27:31,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:31,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:31,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:31,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:31,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451991) roll requested 2024-11-13T18:27:31,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522451999 2024-11-13T18:27:32,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451999 2024-11-13T18:27:32,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522451999) roll requested 2024-11-13T18:27:32,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452005 2024-11-13T18:27:32,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522451999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452005 2024-11-13T18:27:32,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452005) roll requested 2024-11-13T18:27:32,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452012 2024-11-13T18:27:32,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452012 2024-11-13T18:27:32,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452012) roll requested 2024-11-13T18:27:32,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452020 2024-11-13T18:27:32,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452020 2024-11-13T18:27:32,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452020) roll requested 2024-11-13T18:27:32,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452030 2024-11-13T18:27:32,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452030 2024-11-13T18:27:32,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452030) roll requested 2024-11-13T18:27:32,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452038 2024-11-13T18:27:32,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452038 2024-11-13T18:27:32,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452038) roll requested 2024-11-13T18:27:32,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452050 2024-11-13T18:27:32,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452050 2024-11-13T18:27:32,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452050) roll requested 2024-11-13T18:27:32,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452067 2024-11-13T18:27:32,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452067 2024-11-13T18:27:32,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452067) roll requested 2024-11-13T18:27:32,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452101 2024-11-13T18:27:32,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452101 2024-11-13T18:27:32,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452101) roll requested 2024-11-13T18:27:32,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452113 2024-11-13T18:27:32,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452113 2024-11-13T18:27:32,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452113) roll requested 2024-11-13T18:27:32,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452128 2024-11-13T18:27:32,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452128 2024-11-13T18:27:32,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452128) roll requested 2024-11-13T18:27:32,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452141 2024-11-13T18:27:32,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452141 2024-11-13T18:27:32,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,158 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,158 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452141) roll requested 2024-11-13T18:27:32,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452158 2024-11-13T18:27:32,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452158 2024-11-13T18:27:32,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452158) roll requested 2024-11-13T18:27:32,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452167 2024-11-13T18:27:32,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452167 2024-11-13T18:27:32,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452167) roll requested 2024-11-13T18:27:32,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452178 2024-11-13T18:27:32,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452178 2024-11-13T18:27:32,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452178) roll requested 2024-11-13T18:27:32,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452197 2024-11-13T18:27:32,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452197 2024-11-13T18:27:32,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452197) roll requested 2024-11-13T18:27:32,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452221 2024-11-13T18:27:32,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452221 2024-11-13T18:27:32,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452221) roll requested 2024-11-13T18:27:32,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452233 2024-11-13T18:27:32,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452233 2024-11-13T18:27:32,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452233) roll requested 2024-11-13T18:27:32,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452246 2024-11-13T18:27:32,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452246 2024-11-13T18:27:32,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452246) roll requested 2024-11-13T18:27:32,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452257 2024-11-13T18:27:32,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452257 2024-11-13T18:27:32,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452257) roll requested 2024-11-13T18:27:32,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452268 2024-11-13T18:27:32,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452268 2024-11-13T18:27:32,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452268) roll requested 2024-11-13T18:27:32,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452284 2024-11-13T18:27:32,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452284 2024-11-13T18:27:32,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452284) roll requested 2024-11-13T18:27:32,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452294 2024-11-13T18:27:32,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452294 2024-11-13T18:27:32,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452294) roll requested 2024-11-13T18:27:32,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452305 2024-11-13T18:27:32,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452305 2024-11-13T18:27:32,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452305) roll requested 2024-11-13T18:27:32,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452313 2024-11-13T18:27:32,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452313 2024-11-13T18:27:32,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452313) roll requested 2024-11-13T18:27:32,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452321 2024-11-13T18:27:32,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452321 2024-11-13T18:27:32,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452321) roll requested 2024-11-13T18:27:32,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452330 2024-11-13T18:27:32,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452330 2024-11-13T18:27:32,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452330) roll requested 2024-11-13T18:27:32,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452349 2024-11-13T18:27:32,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452349 2024-11-13T18:27:32,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452349) roll requested 2024-11-13T18:27:32,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452361 2024-11-13T18:27:32,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452361 2024-11-13T18:27:32,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452361) roll requested 2024-11-13T18:27:32,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452372 2024-11-13T18:27:32,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452372 2024-11-13T18:27:32,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452372) roll requested 2024-11-13T18:27:32,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452385 2024-11-13T18:27:32,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452372 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452385 2024-11-13T18:27:32,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452385) roll requested 2024-11-13T18:27:32,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452395 2024-11-13T18:27:32,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452395 2024-11-13T18:27:32,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452395) roll requested 2024-11-13T18:27:32,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452403 2024-11-13T18:27:32,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452403 2024-11-13T18:27:32,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452403) roll requested 2024-11-13T18:27:32,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452410 2024-11-13T18:27:32,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452410 2024-11-13T18:27:32,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452410) roll requested 2024-11-13T18:27:32,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452417 2024-11-13T18:27:32,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452417 2024-11-13T18:27:32,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452417) roll requested 2024-11-13T18:27:32,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452431 2024-11-13T18:27:32,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452431 2024-11-13T18:27:32,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452431) roll requested 2024-11-13T18:27:32,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452440 2024-11-13T18:27:32,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452440 2024-11-13T18:27:32,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452440) roll requested 2024-11-13T18:27:32,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452449 2024-11-13T18:27:32,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452449 2024-11-13T18:27:32,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452449) roll requested 2024-11-13T18:27:32,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452457 2024-11-13T18:27:32,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452457 2024-11-13T18:27:32,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452457) roll requested 2024-11-13T18:27:32,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452468 2024-11-13T18:27:32,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452468 2024-11-13T18:27:32,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452468) roll requested 2024-11-13T18:27:32,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452476 2024-11-13T18:27:32,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452476 2024-11-13T18:27:32,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452476) roll requested 2024-11-13T18:27:32,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452483 2024-11-13T18:27:32,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452483 2024-11-13T18:27:32,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452483) roll requested 2024-11-13T18:27:32,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452493 2024-11-13T18:27:32,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452493 2024-11-13T18:27:32,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452493) roll requested 2024-11-13T18:27:32,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452500 2024-11-13T18:27:32,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452500 2024-11-13T18:27:32,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452500) roll requested 2024-11-13T18:27:32,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452510 2024-11-13T18:27:32,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452510 2024-11-13T18:27:32,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452510) roll requested 2024-11-13T18:27:32,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452521 2024-11-13T18:27:32,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452521 2024-11-13T18:27:32,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452521) roll requested 2024-11-13T18:27:32,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452532 2024-11-13T18:27:32,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452532 2024-11-13T18:27:32,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452532) roll requested 2024-11-13T18:27:32,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452544 2024-11-13T18:27:32,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452544 2024-11-13T18:27:32,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452544) roll requested 2024-11-13T18:27:32,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452551 2024-11-13T18:27:32,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452551 2024-11-13T18:27:32,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452551) roll requested 2024-11-13T18:27:32,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452558 2024-11-13T18:27:32,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452558 2024-11-13T18:27:32,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452558) roll requested 2024-11-13T18:27:32,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452565 2024-11-13T18:27:32,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452565 2024-11-13T18:27:32,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452565) roll requested 2024-11-13T18:27:32,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452573 2024-11-13T18:27:32,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452573 2024-11-13T18:27:32,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452573) roll requested 2024-11-13T18:27:32,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452580 2024-11-13T18:27:32,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452580 2024-11-13T18:27:32,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452580) roll requested 2024-11-13T18:27:32,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452587 2024-11-13T18:27:32,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452587 2024-11-13T18:27:32,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452587) roll requested 2024-11-13T18:27:32,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452595 2024-11-13T18:27:32,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452595 2024-11-13T18:27:32,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452595) roll requested 2024-11-13T18:27:32,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452602 2024-11-13T18:27:32,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452602 2024-11-13T18:27:32,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452602) roll requested 2024-11-13T18:27:32,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452610 2024-11-13T18:27:32,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452610 2024-11-13T18:27:32,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452610) roll requested 2024-11-13T18:27:32,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452617 2024-11-13T18:27:32,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452617 2024-11-13T18:27:32,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452617) roll requested 2024-11-13T18:27:32,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452625 2024-11-13T18:27:32,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452625 2024-11-13T18:27:32,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452625) roll requested 2024-11-13T18:27:32,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452633 2024-11-13T18:27:32,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452633 2024-11-13T18:27:32,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452633) roll requested 2024-11-13T18:27:32,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452640 2024-11-13T18:27:32,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452640 2024-11-13T18:27:32,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452640) roll requested 2024-11-13T18:27:32,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452649 2024-11-13T18:27:32,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452649 2024-11-13T18:27:32,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452649) roll requested 2024-11-13T18:27:32,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452658 2024-11-13T18:27:32,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452658 2024-11-13T18:27:32,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452658) roll requested 2024-11-13T18:27:32,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452665 2024-11-13T18:27:32,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452665 2024-11-13T18:27:32,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452665) roll requested 2024-11-13T18:27:32,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452673 2024-11-13T18:27:32,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452673 2024-11-13T18:27:32,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452673) roll requested 2024-11-13T18:27:32,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452681 2024-11-13T18:27:32,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452681 2024-11-13T18:27:32,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452681) roll requested 2024-11-13T18:27:32,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452689 2024-11-13T18:27:32,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452689 2024-11-13T18:27:32,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452689) roll requested 2024-11-13T18:27:32,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452701 2024-11-13T18:27:32,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452701 2024-11-13T18:27:32,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452701) roll requested 2024-11-13T18:27:32,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452710 2024-11-13T18:27:32,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452710 2024-11-13T18:27:32,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452710) roll requested 2024-11-13T18:27:32,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452717 2024-11-13T18:27:32,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452717 2024-11-13T18:27:32,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452717) roll requested 2024-11-13T18:27:32,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452736 2024-11-13T18:27:32,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452736 2024-11-13T18:27:32,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452736) roll requested 2024-11-13T18:27:32,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452745 2024-11-13T18:27:32,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452745 2024-11-13T18:27:32,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452745) roll requested 2024-11-13T18:27:32,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452757 2024-11-13T18:27:32,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452757 2024-11-13T18:27:32,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452757) roll requested 2024-11-13T18:27:32,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452765 2024-11-13T18:27:32,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452765 2024-11-13T18:27:32,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452765) roll requested 2024-11-13T18:27:32,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452774 2024-11-13T18:27:32,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452774 2024-11-13T18:27:32,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452774) roll requested 2024-11-13T18:27:32,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452782 2024-11-13T18:27:32,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452782 2024-11-13T18:27:32,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452782) roll requested 2024-11-13T18:27:32,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452793 2024-11-13T18:27:32,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452793 2024-11-13T18:27:32,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452793) roll requested 2024-11-13T18:27:32,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452801 2024-11-13T18:27:32,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452801 2024-11-13T18:27:32,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452801) roll requested 2024-11-13T18:27:32,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452812 2024-11-13T18:27:32,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452812 2024-11-13T18:27:32,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452812) roll requested 2024-11-13T18:27:32,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452820 2024-11-13T18:27:32,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452820 2024-11-13T18:27:32,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452820) roll requested 2024-11-13T18:27:32,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452828 2024-11-13T18:27:32,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452828 2024-11-13T18:27:32,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452828) roll requested 2024-11-13T18:27:32,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452837 2024-11-13T18:27:32,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452837 2024-11-13T18:27:32,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452837) roll requested 2024-11-13T18:27:32,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452849 2024-11-13T18:27:32,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452849 2024-11-13T18:27:32,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452849) roll requested 2024-11-13T18:27:32,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452861 2024-11-13T18:27:32,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452861 2024-11-13T18:27:32,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452861) roll requested 2024-11-13T18:27:32,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452871 2024-11-13T18:27:32,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452871 2024-11-13T18:27:32,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452871) roll requested 2024-11-13T18:27:32,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452880 2024-11-13T18:27:32,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452880 2024-11-13T18:27:32,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452880) roll requested 2024-11-13T18:27:32,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452889 2024-11-13T18:27:32,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452889 2024-11-13T18:27:32,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452889) roll requested 2024-11-13T18:27:32,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452902 2024-11-13T18:27:32,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452902 2024-11-13T18:27:32,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452902) roll requested 2024-11-13T18:27:32,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452917 2024-11-13T18:27:32,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452917 2024-11-13T18:27:32,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452917) roll requested 2024-11-13T18:27:32,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452929 2024-11-13T18:27:32,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452929 2024-11-13T18:27:32,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452929) roll requested 2024-11-13T18:27:32,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452941 2024-11-13T18:27:32,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452941 2024-11-13T18:27:32,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452941) roll requested 2024-11-13T18:27:32,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452952 2024-11-13T18:27:32,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452952 2024-11-13T18:27:32,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452952) roll requested 2024-11-13T18:27:32,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452960 2024-11-13T18:27:32,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452960 2024-11-13T18:27:32,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452960) roll requested 2024-11-13T18:27:32,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452968 2024-11-13T18:27:32,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452968 2024-11-13T18:27:32,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452968) roll requested 2024-11-13T18:27:32,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452976 2024-11-13T18:27:32,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452976 2024-11-13T18:27:32,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452976) roll requested 2024-11-13T18:27:32,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452983 2024-11-13T18:27:32,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452983 2024-11-13T18:27:32,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:32,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:32,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452983) roll requested 2024-11-13T18:27:32,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522452992 2024-11-13T18:27:32,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:32,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452992 2024-11-13T18:27:32,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:32,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:32,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522452992) roll requested 2024-11-13T18:27:33,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453000 2024-11-13T18:27:33,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522452992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453000 2024-11-13T18:27:33,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453000) roll requested 2024-11-13T18:27:33,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453007 2024-11-13T18:27:33,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453007 2024-11-13T18:27:33,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453007) roll requested 2024-11-13T18:27:33,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453015 2024-11-13T18:27:33,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453015 2024-11-13T18:27:33,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453015) roll requested 2024-11-13T18:27:33,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453021 2024-11-13T18:27:33,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453021 2024-11-13T18:27:33,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453021) roll requested 2024-11-13T18:27:33,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453029 2024-11-13T18:27:33,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453029 2024-11-13T18:27:33,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453029) roll requested 2024-11-13T18:27:33,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453037 2024-11-13T18:27:33,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453037 2024-11-13T18:27:33,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453037) roll requested 2024-11-13T18:27:33,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453044 2024-11-13T18:27:33,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453044 2024-11-13T18:27:33,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453044) roll requested 2024-11-13T18:27:33,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453051 2024-11-13T18:27:33,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453051 2024-11-13T18:27:33,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453051) roll requested 2024-11-13T18:27:33,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453059 2024-11-13T18:27:33,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453059 2024-11-13T18:27:33,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453059) roll requested 2024-11-13T18:27:33,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453067 2024-11-13T18:27:33,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453067 2024-11-13T18:27:33,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453067) roll requested 2024-11-13T18:27:33,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453074 2024-11-13T18:27:33,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453074 2024-11-13T18:27:33,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453074) roll requested 2024-11-13T18:27:33,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453082 2024-11-13T18:27:33,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453082 2024-11-13T18:27:33,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453082) roll requested 2024-11-13T18:27:33,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453089 2024-11-13T18:27:33,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453089 2024-11-13T18:27:33,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453089) roll requested 2024-11-13T18:27:33,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453097 2024-11-13T18:27:33,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453097 2024-11-13T18:27:33,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453097) roll requested 2024-11-13T18:27:33,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453104 2024-11-13T18:27:33,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453104 2024-11-13T18:27:33,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453104) roll requested 2024-11-13T18:27:33,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453111 2024-11-13T18:27:33,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453111 2024-11-13T18:27:33,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453111) roll requested 2024-11-13T18:27:33,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453118 2024-11-13T18:27:33,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453118 2024-11-13T18:27:33,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453118) roll requested 2024-11-13T18:27:33,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453143 2024-11-13T18:27:33,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453143 2024-11-13T18:27:33,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453143) roll requested 2024-11-13T18:27:33,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453151 2024-11-13T18:27:33,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453151 2024-11-13T18:27:33,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453151) roll requested 2024-11-13T18:27:33,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453159 2024-11-13T18:27:33,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453159 2024-11-13T18:27:33,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453159) roll requested 2024-11-13T18:27:33,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453166 2024-11-13T18:27:33,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453166 2024-11-13T18:27:33,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453166) roll requested 2024-11-13T18:27:33,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453173 2024-11-13T18:27:33,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453173 2024-11-13T18:27:33,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453173) roll requested 2024-11-13T18:27:33,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453180 2024-11-13T18:27:33,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453180 2024-11-13T18:27:33,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453180) roll requested 2024-11-13T18:27:33,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453188 2024-11-13T18:27:33,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453188 2024-11-13T18:27:33,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453188) roll requested 2024-11-13T18:27:33,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453196 2024-11-13T18:27:33,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453196 2024-11-13T18:27:33,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453196) roll requested 2024-11-13T18:27:33,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453204 2024-11-13T18:27:33,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453204 2024-11-13T18:27:33,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453204) roll requested 2024-11-13T18:27:33,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453222 2024-11-13T18:27:33,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453222 2024-11-13T18:27:33,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453222) roll requested 2024-11-13T18:27:33,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453246 2024-11-13T18:27:33,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453246 2024-11-13T18:27:33,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453246) roll requested 2024-11-13T18:27:33,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453276 2024-11-13T18:27:33,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453276 2024-11-13T18:27:33,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453276) roll requested 2024-11-13T18:27:33,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453321 2024-11-13T18:27:33,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453321 2024-11-13T18:27:33,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453321) roll requested 2024-11-13T18:27:33,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453339 2024-11-13T18:27:33,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453339 2024-11-13T18:27:33,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453339) roll requested 2024-11-13T18:27:33,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453360 2024-11-13T18:27:33,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453360 2024-11-13T18:27:33,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453360) roll requested 2024-11-13T18:27:33,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453368 2024-11-13T18:27:33,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453368 2024-11-13T18:27:33,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453368) roll requested 2024-11-13T18:27:33,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453376 2024-11-13T18:27:33,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453376 2024-11-13T18:27:33,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453376) roll requested 2024-11-13T18:27:33,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453389 2024-11-13T18:27:33,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453389 2024-11-13T18:27:33,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453389) roll requested 2024-11-13T18:27:33,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453397 2024-11-13T18:27:33,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453397 2024-11-13T18:27:33,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453397) roll requested 2024-11-13T18:27:33,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453410 2024-11-13T18:27:33,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453410 2024-11-13T18:27:33,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453410) roll requested 2024-11-13T18:27:33,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453421 2024-11-13T18:27:33,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453421 2024-11-13T18:27:33,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453421) roll requested 2024-11-13T18:27:33,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453428 2024-11-13T18:27:33,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453428 2024-11-13T18:27:33,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453428) roll requested 2024-11-13T18:27:33,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453440 2024-11-13T18:27:33,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453440 2024-11-13T18:27:33,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453440) roll requested 2024-11-13T18:27:33,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453448 2024-11-13T18:27:33,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453448 2024-11-13T18:27:33,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453448) roll requested 2024-11-13T18:27:33,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453459 2024-11-13T18:27:33,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453459 2024-11-13T18:27:33,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453459) roll requested 2024-11-13T18:27:33,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453466 2024-11-13T18:27:33,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453466 2024-11-13T18:27:33,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453466) roll requested 2024-11-13T18:27:33,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453473 2024-11-13T18:27:33,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453473 2024-11-13T18:27:33,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453473) roll requested 2024-11-13T18:27:33,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453480 2024-11-13T18:27:33,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453480 2024-11-13T18:27:33,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453480) roll requested 2024-11-13T18:27:33,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453489 2024-11-13T18:27:33,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453489 2024-11-13T18:27:33,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453489) roll requested 2024-11-13T18:27:33,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453503 2024-11-13T18:27:33,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453503 2024-11-13T18:27:33,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453503) roll requested 2024-11-13T18:27:33,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453527 2024-11-13T18:27:33,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453527 2024-11-13T18:27:33,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453527) roll requested 2024-11-13T18:27:33,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453552 2024-11-13T18:27:33,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453552 2024-11-13T18:27:33,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453552) roll requested 2024-11-13T18:27:33,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453581 2024-11-13T18:27:33,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453581 2024-11-13T18:27:33,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453581) roll requested 2024-11-13T18:27:33,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453617 2024-11-13T18:27:33,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453617 2024-11-13T18:27:33,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453617) roll requested 2024-11-13T18:27:33,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453649 2024-11-13T18:27:33,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453649 2024-11-13T18:27:33,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453649) roll requested 2024-11-13T18:27:33,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453665 2024-11-13T18:27:33,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453665 2024-11-13T18:27:33,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453665) roll requested 2024-11-13T18:27:33,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453672 2024-11-13T18:27:33,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453672 2024-11-13T18:27:33,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453672) roll requested 2024-11-13T18:27:33,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453681 2024-11-13T18:27:33,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453681 2024-11-13T18:27:33,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453681) roll requested 2024-11-13T18:27:33,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453689 2024-11-13T18:27:33,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453689 2024-11-13T18:27:33,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453689) roll requested 2024-11-13T18:27:33,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453697 2024-11-13T18:27:33,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453697 2024-11-13T18:27:33,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453697) roll requested 2024-11-13T18:27:33,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453705 2024-11-13T18:27:33,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453705 2024-11-13T18:27:33,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453705) roll requested 2024-11-13T18:27:33,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453713 2024-11-13T18:27:33,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453713 2024-11-13T18:27:33,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453713) roll requested 2024-11-13T18:27:33,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453721 2024-11-13T18:27:33,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453721 2024-11-13T18:27:33,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453721) roll requested 2024-11-13T18:27:33,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453728 2024-11-13T18:27:33,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453728 2024-11-13T18:27:33,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453728) roll requested 2024-11-13T18:27:33,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453735 2024-11-13T18:27:33,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453735 2024-11-13T18:27:33,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453735) roll requested 2024-11-13T18:27:33,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453742 2024-11-13T18:27:33,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453742 2024-11-13T18:27:33,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453742) roll requested 2024-11-13T18:27:33,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453753 2024-11-13T18:27:33,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453753 2024-11-13T18:27:33,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453753) roll requested 2024-11-13T18:27:33,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453761 2024-11-13T18:27:33,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453761 2024-11-13T18:27:33,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453761) roll requested 2024-11-13T18:27:33,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453769 2024-11-13T18:27:33,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453769 2024-11-13T18:27:33,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453769) roll requested 2024-11-13T18:27:33,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453777 2024-11-13T18:27:33,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453777 2024-11-13T18:27:33,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453777) roll requested 2024-11-13T18:27:33,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453786 2024-11-13T18:27:33,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453786 2024-11-13T18:27:33,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453786) roll requested 2024-11-13T18:27:33,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453794 2024-11-13T18:27:33,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453794 2024-11-13T18:27:33,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453794) roll requested 2024-11-13T18:27:33,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453800 2024-11-13T18:27:33,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453800 2024-11-13T18:27:33,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453800) roll requested 2024-11-13T18:27:33,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453807 2024-11-13T18:27:33,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453807 2024-11-13T18:27:33,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453807) roll requested 2024-11-13T18:27:33,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453815 2024-11-13T18:27:33,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453815 2024-11-13T18:27:33,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453815) roll requested 2024-11-13T18:27:33,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453822 2024-11-13T18:27:33,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453822 2024-11-13T18:27:33,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453822) roll requested 2024-11-13T18:27:33,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453830 2024-11-13T18:27:33,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453830 2024-11-13T18:27:33,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453830) roll requested 2024-11-13T18:27:33,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453838 2024-11-13T18:27:33,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453838 2024-11-13T18:27:33,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453838) roll requested 2024-11-13T18:27:33,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453846 2024-11-13T18:27:33,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453846 2024-11-13T18:27:33,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453846) roll requested 2024-11-13T18:27:33,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453853 2024-11-13T18:27:33,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453853 2024-11-13T18:27:33,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453853) roll requested 2024-11-13T18:27:33,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453861 2024-11-13T18:27:33,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453861 2024-11-13T18:27:33,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453861) roll requested 2024-11-13T18:27:33,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453869 2024-11-13T18:27:33,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453869 2024-11-13T18:27:33,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453869) roll requested 2024-11-13T18:27:33,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453877 2024-11-13T18:27:33,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453877 2024-11-13T18:27:33,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453877) roll requested 2024-11-13T18:27:33,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453885 2024-11-13T18:27:33,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453885 2024-11-13T18:27:33,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453885) roll requested 2024-11-13T18:27:33,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453893 2024-11-13T18:27:33,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453893 2024-11-13T18:27:33,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453893) roll requested 2024-11-13T18:27:33,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453905 2024-11-13T18:27:33,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453905 2024-11-13T18:27:33,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453905) roll requested 2024-11-13T18:27:33,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453921 2024-11-13T18:27:33,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453921 2024-11-13T18:27:33,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453921) roll requested 2024-11-13T18:27:33,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453929 2024-11-13T18:27:33,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453929 2024-11-13T18:27:33,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453929) roll requested 2024-11-13T18:27:33,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453940 2024-11-13T18:27:33,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453940 2024-11-13T18:27:33,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453940) roll requested 2024-11-13T18:27:33,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453947 2024-11-13T18:27:33,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453947 2024-11-13T18:27:33,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453947) roll requested 2024-11-13T18:27:33,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453956 2024-11-13T18:27:33,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453956 2024-11-13T18:27:33,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453956) roll requested 2024-11-13T18:27:33,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453964 2024-11-13T18:27:33,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453964 2024-11-13T18:27:33,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453964) roll requested 2024-11-13T18:27:33,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453971 2024-11-13T18:27:33,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453971 2024-11-13T18:27:33,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453971) roll requested 2024-11-13T18:27:33,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453978 2024-11-13T18:27:33,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453978 2024-11-13T18:27:33,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453978) roll requested 2024-11-13T18:27:33,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453986 2024-11-13T18:27:33,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453986 2024-11-13T18:27:33,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453986) roll requested 2024-11-13T18:27:33,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453992 2024-11-13T18:27:33,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:33,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453992 2024-11-13T18:27:33,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:33,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:33,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:33,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:33,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453992) roll requested 2024-11-13T18:27:33,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522453999 2024-11-13T18:27:34,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453999 2024-11-13T18:27:34,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522453999) roll requested 2024-11-13T18:27:34,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454007 2024-11-13T18:27:34,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522453999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454007 2024-11-13T18:27:34,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454007) roll requested 2024-11-13T18:27:34,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454016 2024-11-13T18:27:34,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454016 2024-11-13T18:27:34,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454016) roll requested 2024-11-13T18:27:34,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454023 2024-11-13T18:27:34,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454016 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454023 2024-11-13T18:27:34,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454023) roll requested 2024-11-13T18:27:34,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454030 2024-11-13T18:27:34,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454030 2024-11-13T18:27:34,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454030) roll requested 2024-11-13T18:27:34,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454038 2024-11-13T18:27:34,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454038 2024-11-13T18:27:34,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454038) roll requested 2024-11-13T18:27:34,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454046 2024-11-13T18:27:34,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454046 2024-11-13T18:27:34,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454046) roll requested 2024-11-13T18:27:34,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454054 2024-11-13T18:27:34,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454054 2024-11-13T18:27:34,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454054) roll requested 2024-11-13T18:27:34,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454064 2024-11-13T18:27:34,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454064 2024-11-13T18:27:34,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454064) roll requested 2024-11-13T18:27:34,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454072 2024-11-13T18:27:34,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454072 2024-11-13T18:27:34,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454072) roll requested 2024-11-13T18:27:34,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454080 2024-11-13T18:27:34,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454080 2024-11-13T18:27:34,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454080) roll requested 2024-11-13T18:27:34,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454100 2024-11-13T18:27:34,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454100 2024-11-13T18:27:34,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454100) roll requested 2024-11-13T18:27:34,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454109 2024-11-13T18:27:34,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454109 2024-11-13T18:27:34,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454109) roll requested 2024-11-13T18:27:34,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454121 2024-11-13T18:27:34,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454121 2024-11-13T18:27:34,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454121) roll requested 2024-11-13T18:27:34,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454130 2024-11-13T18:27:34,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454130 2024-11-13T18:27:34,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454130) roll requested 2024-11-13T18:27:34,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454141 2024-11-13T18:27:34,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454141 2024-11-13T18:27:34,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454141) roll requested 2024-11-13T18:27:34,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454153 2024-11-13T18:27:34,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454153 2024-11-13T18:27:34,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454153) roll requested 2024-11-13T18:27:34,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454161 2024-11-13T18:27:34,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454161 2024-11-13T18:27:34,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454161) roll requested 2024-11-13T18:27:34,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454171 2024-11-13T18:27:34,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454171 2024-11-13T18:27:34,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454171) roll requested 2024-11-13T18:27:34,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454178 2024-11-13T18:27:34,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454178 2024-11-13T18:27:34,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454178) roll requested 2024-11-13T18:27:34,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454186 2024-11-13T18:27:34,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454186 2024-11-13T18:27:34,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454186) roll requested 2024-11-13T18:27:34,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454194 2024-11-13T18:27:34,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454194 2024-11-13T18:27:34,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454194) roll requested 2024-11-13T18:27:34,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454201 2024-11-13T18:27:34,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454201 2024-11-13T18:27:34,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454201) roll requested 2024-11-13T18:27:34,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454209 2024-11-13T18:27:34,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454209 2024-11-13T18:27:34,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454209) roll requested 2024-11-13T18:27:34,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454217 2024-11-13T18:27:34,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454217 2024-11-13T18:27:34,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454217) roll requested 2024-11-13T18:27:34,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454226 2024-11-13T18:27:34,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454226 2024-11-13T18:27:34,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454226) roll requested 2024-11-13T18:27:34,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454235 2024-11-13T18:27:34,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454235 2024-11-13T18:27:34,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454235) roll requested 2024-11-13T18:27:34,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454245 2024-11-13T18:27:34,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454245 2024-11-13T18:27:34,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454245) roll requested 2024-11-13T18:27:34,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454255 2024-11-13T18:27:34,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454255 2024-11-13T18:27:34,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454255) roll requested 2024-11-13T18:27:34,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454272 2024-11-13T18:27:34,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454272 2024-11-13T18:27:34,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454272) roll requested 2024-11-13T18:27:34,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454280 2024-11-13T18:27:34,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454280 2024-11-13T18:27:34,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454280) roll requested 2024-11-13T18:27:34,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454287 2024-11-13T18:27:34,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454287 2024-11-13T18:27:34,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454287) roll requested 2024-11-13T18:27:34,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454293 2024-11-13T18:27:34,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454293 2024-11-13T18:27:34,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454293) roll requested 2024-11-13T18:27:34,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454300 2024-11-13T18:27:34,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454300 2024-11-13T18:27:34,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454300) roll requested 2024-11-13T18:27:34,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454309 2024-11-13T18:27:34,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454309 2024-11-13T18:27:34,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454309) roll requested 2024-11-13T18:27:34,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454316 2024-11-13T18:27:34,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454316 2024-11-13T18:27:34,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454316) roll requested 2024-11-13T18:27:34,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454323 2024-11-13T18:27:34,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454323 2024-11-13T18:27:34,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454323) roll requested 2024-11-13T18:27:34,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454331 2024-11-13T18:27:34,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454331 2024-11-13T18:27:34,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454331) roll requested 2024-11-13T18:27:34,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454339 2024-11-13T18:27:34,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454339 2024-11-13T18:27:34,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454339) roll requested 2024-11-13T18:27:34,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454346 2024-11-13T18:27:34,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454346 2024-11-13T18:27:34,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454346) roll requested 2024-11-13T18:27:34,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454355 2024-11-13T18:27:34,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454355 2024-11-13T18:27:34,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454355) roll requested 2024-11-13T18:27:34,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454363 2024-11-13T18:27:34,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454363 2024-11-13T18:27:34,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454363) roll requested 2024-11-13T18:27:34,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454370 2024-11-13T18:27:34,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454370 2024-11-13T18:27:34,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454370) roll requested 2024-11-13T18:27:34,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454381 2024-11-13T18:27:34,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454381 2024-11-13T18:27:34,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454381) roll requested 2024-11-13T18:27:34,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454390 2024-11-13T18:27:34,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454390 2024-11-13T18:27:34,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454390) roll requested 2024-11-13T18:27:34,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454405 2024-11-13T18:27:34,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454405 2024-11-13T18:27:34,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454405) roll requested 2024-11-13T18:27:34,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454416 2024-11-13T18:27:34,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454416 2024-11-13T18:27:34,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454416) roll requested 2024-11-13T18:27:34,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454444 2024-11-13T18:27:34,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454444 2024-11-13T18:27:34,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454444) roll requested 2024-11-13T18:27:34,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454473 2024-11-13T18:27:34,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454473 2024-11-13T18:27:34,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454473) roll requested 2024-11-13T18:27:34,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454497 2024-11-13T18:27:34,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454497 2024-11-13T18:27:34,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454497) roll requested 2024-11-13T18:27:34,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454508 2024-11-13T18:27:34,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454508 2024-11-13T18:27:34,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454508) roll requested 2024-11-13T18:27:34,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454516 2024-11-13T18:27:34,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454516 2024-11-13T18:27:34,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454516) roll requested 2024-11-13T18:27:34,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454523 2024-11-13T18:27:34,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454523 2024-11-13T18:27:34,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454523) roll requested 2024-11-13T18:27:34,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454531 2024-11-13T18:27:34,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454531 2024-11-13T18:27:34,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454531) roll requested 2024-11-13T18:27:34,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454539 2024-11-13T18:27:34,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454539 2024-11-13T18:27:34,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454539) roll requested 2024-11-13T18:27:34,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454546 2024-11-13T18:27:34,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454546 2024-11-13T18:27:34,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454546) roll requested 2024-11-13T18:27:34,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454553 2024-11-13T18:27:34,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454553 2024-11-13T18:27:34,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454553) roll requested 2024-11-13T18:27:34,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454561 2024-11-13T18:27:34,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454561 2024-11-13T18:27:34,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454561) roll requested 2024-11-13T18:27:34,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454568 2024-11-13T18:27:34,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454568 2024-11-13T18:27:34,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454568) roll requested 2024-11-13T18:27:34,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454575 2024-11-13T18:27:34,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454575 2024-11-13T18:27:34,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454575) roll requested 2024-11-13T18:27:34,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454583 2024-11-13T18:27:34,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454583 2024-11-13T18:27:34,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454583) roll requested 2024-11-13T18:27:34,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454593 2024-11-13T18:27:34,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454593 2024-11-13T18:27:34,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454593) roll requested 2024-11-13T18:27:34,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454601 2024-11-13T18:27:34,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454601 2024-11-13T18:27:34,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454601) roll requested 2024-11-13T18:27:34,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454610 2024-11-13T18:27:34,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454610 2024-11-13T18:27:34,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454610) roll requested 2024-11-13T18:27:34,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454621 2024-11-13T18:27:34,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454621 2024-11-13T18:27:34,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454621) roll requested 2024-11-13T18:27:34,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454628 2024-11-13T18:27:34,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454628 2024-11-13T18:27:34,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454628) roll requested 2024-11-13T18:27:34,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454634 2024-11-13T18:27:34,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454634 2024-11-13T18:27:34,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454634) roll requested 2024-11-13T18:27:34,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454641 2024-11-13T18:27:34,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454641 2024-11-13T18:27:34,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454641) roll requested 2024-11-13T18:27:34,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454648 2024-11-13T18:27:34,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454648 2024-11-13T18:27:34,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454648) roll requested 2024-11-13T18:27:34,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454655 2024-11-13T18:27:34,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454655 2024-11-13T18:27:34,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454655) roll requested 2024-11-13T18:27:34,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454663 2024-11-13T18:27:34,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454663 2024-11-13T18:27:34,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454663) roll requested 2024-11-13T18:27:34,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454674 2024-11-13T18:27:34,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454674 2024-11-13T18:27:34,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454674) roll requested 2024-11-13T18:27:34,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454683 2024-11-13T18:27:34,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454683 2024-11-13T18:27:34,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454683) roll requested 2024-11-13T18:27:34,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454693 2024-11-13T18:27:34,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454693 2024-11-13T18:27:34,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454693) roll requested 2024-11-13T18:27:34,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454700 2024-11-13T18:27:34,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454700 2024-11-13T18:27:34,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454700) roll requested 2024-11-13T18:27:34,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454710 2024-11-13T18:27:34,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454710 2024-11-13T18:27:34,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454710) roll requested 2024-11-13T18:27:34,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454717 2024-11-13T18:27:34,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454717 2024-11-13T18:27:34,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454717) roll requested 2024-11-13T18:27:34,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454724 2024-11-13T18:27:34,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454724 2024-11-13T18:27:34,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454724) roll requested 2024-11-13T18:27:34,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454731 2024-11-13T18:27:34,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454731 2024-11-13T18:27:34,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454731) roll requested 2024-11-13T18:27:34,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454737 2024-11-13T18:27:34,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454737 2024-11-13T18:27:34,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454737) roll requested 2024-11-13T18:27:34,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454744 2024-11-13T18:27:34,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454744 2024-11-13T18:27:34,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454744) roll requested 2024-11-13T18:27:34,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454751 2024-11-13T18:27:34,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454751 2024-11-13T18:27:34,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454751) roll requested 2024-11-13T18:27:34,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454757 2024-11-13T18:27:34,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454757 2024-11-13T18:27:34,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454757) roll requested 2024-11-13T18:27:34,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454764 2024-11-13T18:27:34,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454764 2024-11-13T18:27:34,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454764) roll requested 2024-11-13T18:27:34,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454771 2024-11-13T18:27:34,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454771 2024-11-13T18:27:34,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454771) roll requested 2024-11-13T18:27:34,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454777 2024-11-13T18:27:34,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454777 2024-11-13T18:27:34,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454777) roll requested 2024-11-13T18:27:34,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454784 2024-11-13T18:27:34,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454784 2024-11-13T18:27:34,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454784) roll requested 2024-11-13T18:27:34,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454791 2024-11-13T18:27:34,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454791 2024-11-13T18:27:34,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454791) roll requested 2024-11-13T18:27:34,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454797 2024-11-13T18:27:34,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454797 2024-11-13T18:27:34,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454797) roll requested 2024-11-13T18:27:34,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454804 2024-11-13T18:27:34,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454804 2024-11-13T18:27:34,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454804) roll requested 2024-11-13T18:27:34,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454810 2024-11-13T18:27:34,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454810 2024-11-13T18:27:34,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454810) roll requested 2024-11-13T18:27:34,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454817 2024-11-13T18:27:34,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454817 2024-11-13T18:27:34,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454817) roll requested 2024-11-13T18:27:34,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454824 2024-11-13T18:27:34,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454824 2024-11-13T18:27:34,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454824) roll requested 2024-11-13T18:27:34,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454830 2024-11-13T18:27:34,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454830 2024-11-13T18:27:34,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454830) roll requested 2024-11-13T18:27:34,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454837 2024-11-13T18:27:34,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454837 2024-11-13T18:27:34,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454837) roll requested 2024-11-13T18:27:34,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454844 2024-11-13T18:27:34,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454844 2024-11-13T18:27:34,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454844) roll requested 2024-11-13T18:27:34,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454850 2024-11-13T18:27:34,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454850 2024-11-13T18:27:34,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454850) roll requested 2024-11-13T18:27:34,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454857 2024-11-13T18:27:34,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454857 2024-11-13T18:27:34,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454857) roll requested 2024-11-13T18:27:34,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454865 2024-11-13T18:27:34,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454865 2024-11-13T18:27:34,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454865) roll requested 2024-11-13T18:27:34,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454872 2024-11-13T18:27:34,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454872 2024-11-13T18:27:34,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454872) roll requested 2024-11-13T18:27:34,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454879 2024-11-13T18:27:34,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454879 2024-11-13T18:27:34,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454879) roll requested 2024-11-13T18:27:34,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454887 2024-11-13T18:27:34,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454887 2024-11-13T18:27:34,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454887) roll requested 2024-11-13T18:27:34,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454909 2024-11-13T18:27:34,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454909 2024-11-13T18:27:34,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454909) roll requested 2024-11-13T18:27:34,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454917 2024-11-13T18:27:34,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454917 2024-11-13T18:27:34,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454917) roll requested 2024-11-13T18:27:34,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454925 2024-11-13T18:27:34,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454925 2024-11-13T18:27:34,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454925) roll requested 2024-11-13T18:27:34,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454933 2024-11-13T18:27:34,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454933 2024-11-13T18:27:34,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454933) roll requested 2024-11-13T18:27:34,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454940 2024-11-13T18:27:34,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454940 2024-11-13T18:27:34,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454940) roll requested 2024-11-13T18:27:34,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454949 2024-11-13T18:27:34,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454949 2024-11-13T18:27:34,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454949) roll requested 2024-11-13T18:27:34,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454957 2024-11-13T18:27:34,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454957 2024-11-13T18:27:34,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454957) roll requested 2024-11-13T18:27:34,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454969 2024-11-13T18:27:34,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454969 2024-11-13T18:27:34,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454969) roll requested 2024-11-13T18:27:34,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454976 2024-11-13T18:27:34,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454976 2024-11-13T18:27:34,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454976) roll requested 2024-11-13T18:27:34,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454983 2024-11-13T18:27:34,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454983 2024-11-13T18:27:34,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454983) roll requested 2024-11-13T18:27:34,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454990 2024-11-13T18:27:34,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:34,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454990 2024-11-13T18:27:34,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:34,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:34,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:34,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:34,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454990) roll requested 2024-11-13T18:27:34,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522454999 2024-11-13T18:27:35,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454999 2024-11-13T18:27:35,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522454999) roll requested 2024-11-13T18:27:35,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455007 2024-11-13T18:27:35,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522454999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455007 2024-11-13T18:27:35,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455007) roll requested 2024-11-13T18:27:35,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455015 2024-11-13T18:27:35,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455015 2024-11-13T18:27:35,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455015) roll requested 2024-11-13T18:27:35,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455022 2024-11-13T18:27:35,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455022 2024-11-13T18:27:35,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455022) roll requested 2024-11-13T18:27:35,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455029 2024-11-13T18:27:35,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455029 2024-11-13T18:27:35,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455029) roll requested 2024-11-13T18:27:35,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455037 2024-11-13T18:27:35,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455037 2024-11-13T18:27:35,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455037) roll requested 2024-11-13T18:27:35,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455044 2024-11-13T18:27:35,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455044 2024-11-13T18:27:35,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455044) roll requested 2024-11-13T18:27:35,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455051 2024-11-13T18:27:35,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455051 2024-11-13T18:27:35,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455051) roll requested 2024-11-13T18:27:35,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455059 2024-11-13T18:27:35,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455059 2024-11-13T18:27:35,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455059) roll requested 2024-11-13T18:27:35,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455066 2024-11-13T18:27:35,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455066 2024-11-13T18:27:35,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455066) roll requested 2024-11-13T18:27:35,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455074 2024-11-13T18:27:35,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455074 2024-11-13T18:27:35,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455074) roll requested 2024-11-13T18:27:35,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455081 2024-11-13T18:27:35,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455081 2024-11-13T18:27:35,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455081) roll requested 2024-11-13T18:27:35,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455089 2024-11-13T18:27:35,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455089 2024-11-13T18:27:35,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455089) roll requested 2024-11-13T18:27:35,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455096 2024-11-13T18:27:35,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455096 2024-11-13T18:27:35,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455096) roll requested 2024-11-13T18:27:35,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455104 2024-11-13T18:27:35,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455104 2024-11-13T18:27:35,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455104) roll requested 2024-11-13T18:27:35,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455110 2024-11-13T18:27:35,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455110 2024-11-13T18:27:35,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455110) roll requested 2024-11-13T18:27:35,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455118 2024-11-13T18:27:35,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455118 2024-11-13T18:27:35,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455118) roll requested 2024-11-13T18:27:35,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455126 2024-11-13T18:27:35,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455126 2024-11-13T18:27:35,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455126) roll requested 2024-11-13T18:27:35,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455134 2024-11-13T18:27:35,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455134 2024-11-13T18:27:35,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455134) roll requested 2024-11-13T18:27:35,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455141 2024-11-13T18:27:35,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455141 2024-11-13T18:27:35,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455141) roll requested 2024-11-13T18:27:35,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455149 2024-11-13T18:27:35,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455149 2024-11-13T18:27:35,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455149) roll requested 2024-11-13T18:27:35,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455157 2024-11-13T18:27:35,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455157 2024-11-13T18:27:35,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455157) roll requested 2024-11-13T18:27:35,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455165 2024-11-13T18:27:35,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455165 2024-11-13T18:27:35,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455165) roll requested 2024-11-13T18:27:35,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455173 2024-11-13T18:27:35,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455173 2024-11-13T18:27:35,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455173) roll requested 2024-11-13T18:27:35,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455180 2024-11-13T18:27:35,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455180 2024-11-13T18:27:35,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455180) roll requested 2024-11-13T18:27:35,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455189 2024-11-13T18:27:35,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455189 2024-11-13T18:27:35,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455189) roll requested 2024-11-13T18:27:35,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455198 2024-11-13T18:27:35,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455198 2024-11-13T18:27:35,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455198) roll requested 2024-11-13T18:27:35,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455205 2024-11-13T18:27:35,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455205 2024-11-13T18:27:35,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455205) roll requested 2024-11-13T18:27:35,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455214 2024-11-13T18:27:35,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455214 2024-11-13T18:27:35,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455214) roll requested 2024-11-13T18:27:35,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455221 2024-11-13T18:27:35,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455221 2024-11-13T18:27:35,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455221) roll requested 2024-11-13T18:27:35,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455230 2024-11-13T18:27:35,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455230 2024-11-13T18:27:35,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455230) roll requested 2024-11-13T18:27:35,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455274 2024-11-13T18:27:35,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455274 2024-11-13T18:27:35,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455274) roll requested 2024-11-13T18:27:35,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455283 2024-11-13T18:27:35,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455283 2024-11-13T18:27:35,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455283) roll requested 2024-11-13T18:27:35,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455297 2024-11-13T18:27:35,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455297 2024-11-13T18:27:35,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455297) roll requested 2024-11-13T18:27:35,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455309 2024-11-13T18:27:35,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455309 2024-11-13T18:27:35,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455309) roll requested 2024-11-13T18:27:35,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455322 2024-11-13T18:27:35,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455322 2024-11-13T18:27:35,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455322) roll requested 2024-11-13T18:27:35,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455333 2024-11-13T18:27:35,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455333 2024-11-13T18:27:35,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455333) roll requested 2024-11-13T18:27:35,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455341 2024-11-13T18:27:35,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455341 2024-11-13T18:27:35,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455341) roll requested 2024-11-13T18:27:35,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455348 2024-11-13T18:27:35,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455348 2024-11-13T18:27:35,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455348) roll requested 2024-11-13T18:27:35,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455355 2024-11-13T18:27:35,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455355 2024-11-13T18:27:35,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455355) roll requested 2024-11-13T18:27:35,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455363 2024-11-13T18:27:35,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455363 2024-11-13T18:27:35,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455363) roll requested 2024-11-13T18:27:35,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455370 2024-11-13T18:27:35,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455370 2024-11-13T18:27:35,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455370) roll requested 2024-11-13T18:27:35,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455379 2024-11-13T18:27:35,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455379 2024-11-13T18:27:35,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455379) roll requested 2024-11-13T18:27:35,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455386 2024-11-13T18:27:35,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455386 2024-11-13T18:27:35,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455386) roll requested 2024-11-13T18:27:35,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455401 2024-11-13T18:27:35,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455401 2024-11-13T18:27:35,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455401) roll requested 2024-11-13T18:27:35,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455415 2024-11-13T18:27:35,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455415 2024-11-13T18:27:35,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455415) roll requested 2024-11-13T18:27:35,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455425 2024-11-13T18:27:35,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455425 2024-11-13T18:27:35,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455425) roll requested 2024-11-13T18:27:35,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455437 2024-11-13T18:27:35,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455437 2024-11-13T18:27:35,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455437) roll requested 2024-11-13T18:27:35,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455446 2024-11-13T18:27:35,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455446 2024-11-13T18:27:35,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455446) roll requested 2024-11-13T18:27:35,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455457 2024-11-13T18:27:35,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455457 2024-11-13T18:27:35,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455457) roll requested 2024-11-13T18:27:35,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455465 2024-11-13T18:27:35,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455465 2024-11-13T18:27:35,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455465) roll requested 2024-11-13T18:27:35,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455473 2024-11-13T18:27:35,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455473 2024-11-13T18:27:35,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455473) roll requested 2024-11-13T18:27:35,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455481 2024-11-13T18:27:35,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455481 2024-11-13T18:27:35,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455481) roll requested 2024-11-13T18:27:35,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455493 2024-11-13T18:27:35,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455493 2024-11-13T18:27:35,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455493) roll requested 2024-11-13T18:27:35,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455504 2024-11-13T18:27:35,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455504 2024-11-13T18:27:35,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455504) roll requested 2024-11-13T18:27:35,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455513 2024-11-13T18:27:35,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455513 2024-11-13T18:27:35,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455513) roll requested 2024-11-13T18:27:35,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455522 2024-11-13T18:27:35,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455522 2024-11-13T18:27:35,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455522) roll requested 2024-11-13T18:27:35,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455529 2024-11-13T18:27:35,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455529 2024-11-13T18:27:35,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455529) roll requested 2024-11-13T18:27:35,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455537 2024-11-13T18:27:35,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455537 2024-11-13T18:27:35,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,545 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455537) roll requested 2024-11-13T18:27:35,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455545 2024-11-13T18:27:35,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455545 2024-11-13T18:27:35,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455545) roll requested 2024-11-13T18:27:35,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455553 2024-11-13T18:27:35,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455553 2024-11-13T18:27:35,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455553) roll requested 2024-11-13T18:27:35,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455561 2024-11-13T18:27:35,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455561 2024-11-13T18:27:35,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455561) roll requested 2024-11-13T18:27:35,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455569 2024-11-13T18:27:35,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455569 2024-11-13T18:27:35,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455569) roll requested 2024-11-13T18:27:35,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455577 2024-11-13T18:27:35,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455577 2024-11-13T18:27:35,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455577) roll requested 2024-11-13T18:27:35,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455585 2024-11-13T18:27:35,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455585 2024-11-13T18:27:35,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455585) roll requested 2024-11-13T18:27:35,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455593 2024-11-13T18:27:35,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455593 2024-11-13T18:27:35,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455593) roll requested 2024-11-13T18:27:35,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455601 2024-11-13T18:27:35,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455601 2024-11-13T18:27:35,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455601) roll requested 2024-11-13T18:27:35,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455609 2024-11-13T18:27:35,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455609 2024-11-13T18:27:35,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455609) roll requested 2024-11-13T18:27:35,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455617 2024-11-13T18:27:35,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455617 2024-11-13T18:27:35,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455617) roll requested 2024-11-13T18:27:35,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455639 2024-11-13T18:27:35,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455639 2024-11-13T18:27:35,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455639) roll requested 2024-11-13T18:27:35,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455682 2024-11-13T18:27:35,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455682 2024-11-13T18:27:35,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455682) roll requested 2024-11-13T18:27:35,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455713 2024-11-13T18:27:35,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455713 2024-11-13T18:27:35,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455713) roll requested 2024-11-13T18:27:35,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455760 2024-11-13T18:27:35,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455760 2024-11-13T18:27:35,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455760) roll requested 2024-11-13T18:27:35,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455817 2024-11-13T18:27:35,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455817 2024-11-13T18:27:35,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455817) roll requested 2024-11-13T18:27:35,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455828 2024-11-13T18:27:35,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455828 2024-11-13T18:27:35,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455828) roll requested 2024-11-13T18:27:35,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455839 2024-11-13T18:27:35,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455839 2024-11-13T18:27:35,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455839) roll requested 2024-11-13T18:27:35,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455846 2024-11-13T18:27:35,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455846 2024-11-13T18:27:35,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455846) roll requested 2024-11-13T18:27:35,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455854 2024-11-13T18:27:35,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455854 2024-11-13T18:27:35,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455854) roll requested 2024-11-13T18:27:35,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455862 2024-11-13T18:27:35,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455862 2024-11-13T18:27:35,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455862) roll requested 2024-11-13T18:27:35,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455870 2024-11-13T18:27:35,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455870 2024-11-13T18:27:35,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455870) roll requested 2024-11-13T18:27:35,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455878 2024-11-13T18:27:35,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455878 2024-11-13T18:27:35,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455878) roll requested 2024-11-13T18:27:35,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455886 2024-11-13T18:27:35,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455886 2024-11-13T18:27:35,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455886) roll requested 2024-11-13T18:27:35,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455893 2024-11-13T18:27:35,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455893 2024-11-13T18:27:35,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455893) roll requested 2024-11-13T18:27:35,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455901 2024-11-13T18:27:35,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455901 2024-11-13T18:27:35,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455901) roll requested 2024-11-13T18:27:35,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455909 2024-11-13T18:27:35,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455909 2024-11-13T18:27:35,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455909) roll requested 2024-11-13T18:27:35,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455916 2024-11-13T18:27:35,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455916 2024-11-13T18:27:35,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455916) roll requested 2024-11-13T18:27:35,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455924 2024-11-13T18:27:35,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455924 2024-11-13T18:27:35,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455924) roll requested 2024-11-13T18:27:35,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455931 2024-11-13T18:27:35,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455931 2024-11-13T18:27:35,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455931) roll requested 2024-11-13T18:27:35,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455938 2024-11-13T18:27:35,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455938 2024-11-13T18:27:35,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455938) roll requested 2024-11-13T18:27:35,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455946 2024-11-13T18:27:35,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455946 2024-11-13T18:27:35,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455946) roll requested 2024-11-13T18:27:35,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455954 2024-11-13T18:27:35,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455954 2024-11-13T18:27:35,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455954) roll requested 2024-11-13T18:27:35,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455965 2024-11-13T18:27:35,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455965 2024-11-13T18:27:35,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:35,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:35,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:35,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:35,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455965) roll requested 2024-11-13T18:27:35,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522455990 2024-11-13T18:27:35,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:35,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455990 2024-11-13T18:27:36,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522455990) roll requested 2024-11-13T18:27:36,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456011 2024-11-13T18:27:36,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522455990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456011 2024-11-13T18:27:36,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456011) roll requested 2024-11-13T18:27:36,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456019 2024-11-13T18:27:36,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456019 2024-11-13T18:27:36,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456019) roll requested 2024-11-13T18:27:36,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456027 2024-11-13T18:27:36,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456027 2024-11-13T18:27:36,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456027) roll requested 2024-11-13T18:27:36,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456034 2024-11-13T18:27:36,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456034 2024-11-13T18:27:36,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456034) roll requested 2024-11-13T18:27:36,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456041 2024-11-13T18:27:36,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456041 2024-11-13T18:27:36,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456041) roll requested 2024-11-13T18:27:36,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456048 2024-11-13T18:27:36,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456048 2024-11-13T18:27:36,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456048) roll requested 2024-11-13T18:27:36,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456055 2024-11-13T18:27:36,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456055 2024-11-13T18:27:36,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456055) roll requested 2024-11-13T18:27:36,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456061 2024-11-13T18:27:36,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456061 2024-11-13T18:27:36,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456061) roll requested 2024-11-13T18:27:36,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456068 2024-11-13T18:27:36,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456068 2024-11-13T18:27:36,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456068) roll requested 2024-11-13T18:27:36,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456075 2024-11-13T18:27:36,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456075 2024-11-13T18:27:36,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456075) roll requested 2024-11-13T18:27:36,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456081 2024-11-13T18:27:36,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456081 2024-11-13T18:27:36,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,088 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456081) roll requested 2024-11-13T18:27:36,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456088 2024-11-13T18:27:36,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456088 2024-11-13T18:27:36,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456088) roll requested 2024-11-13T18:27:36,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456094 2024-11-13T18:27:36,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456094 2024-11-13T18:27:36,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456094) roll requested 2024-11-13T18:27:36,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456106 2024-11-13T18:27:36,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456106 2024-11-13T18:27:36,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456106) roll requested 2024-11-13T18:27:36,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456114 2024-11-13T18:27:36,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456114 2024-11-13T18:27:36,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456114) roll requested 2024-11-13T18:27:36,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456122 2024-11-13T18:27:36,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456122 2024-11-13T18:27:36,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456122) roll requested 2024-11-13T18:27:36,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456129 2024-11-13T18:27:36,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456129 2024-11-13T18:27:36,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456129) roll requested 2024-11-13T18:27:36,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456137 2024-11-13T18:27:36,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456137 2024-11-13T18:27:36,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456137) roll requested 2024-11-13T18:27:36,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456145 2024-11-13T18:27:36,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456145 2024-11-13T18:27:36,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456145) roll requested 2024-11-13T18:27:36,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456153 2024-11-13T18:27:36,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456153 2024-11-13T18:27:36,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456153) roll requested 2024-11-13T18:27:36,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456160 2024-11-13T18:27:36,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456160 2024-11-13T18:27:36,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456160) roll requested 2024-11-13T18:27:36,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456168 2024-11-13T18:27:36,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456168 2024-11-13T18:27:36,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456168) roll requested 2024-11-13T18:27:36,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456177 2024-11-13T18:27:36,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456177 2024-11-13T18:27:36,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456177) roll requested 2024-11-13T18:27:36,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456189 2024-11-13T18:27:36,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456189 2024-11-13T18:27:36,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456189) roll requested 2024-11-13T18:27:36,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456197 2024-11-13T18:27:36,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456197 2024-11-13T18:27:36,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456197) roll requested 2024-11-13T18:27:36,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456209 2024-11-13T18:27:36,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456209 2024-11-13T18:27:36,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456209) roll requested 2024-11-13T18:27:36,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456216 2024-11-13T18:27:36,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456216 2024-11-13T18:27:36,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456216) roll requested 2024-11-13T18:27:36,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456224 2024-11-13T18:27:36,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456224 2024-11-13T18:27:36,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456224) roll requested 2024-11-13T18:27:36,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456237 2024-11-13T18:27:36,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456237 2024-11-13T18:27:36,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456237) roll requested 2024-11-13T18:27:36,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456249 2024-11-13T18:27:36,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456249 2024-11-13T18:27:36,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456249) roll requested 2024-11-13T18:27:36,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456261 2024-11-13T18:27:36,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456261 2024-11-13T18:27:36,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456261) roll requested 2024-11-13T18:27:36,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456270 2024-11-13T18:27:36,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456270 2024-11-13T18:27:36,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456270) roll requested 2024-11-13T18:27:36,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456278 2024-11-13T18:27:36,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456278 2024-11-13T18:27:36,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456278) roll requested 2024-11-13T18:27:36,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456285 2024-11-13T18:27:36,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456285 2024-11-13T18:27:36,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456285) roll requested 2024-11-13T18:27:36,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456293 2024-11-13T18:27:36,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456293 2024-11-13T18:27:36,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456293) roll requested 2024-11-13T18:27:36,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456301 2024-11-13T18:27:36,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456301 2024-11-13T18:27:36,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456301) roll requested 2024-11-13T18:27:36,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456313 2024-11-13T18:27:36,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456313 2024-11-13T18:27:36,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456313) roll requested 2024-11-13T18:27:36,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456325 2024-11-13T18:27:36,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456325 2024-11-13T18:27:36,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456325) roll requested 2024-11-13T18:27:36,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456334 2024-11-13T18:27:36,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456334 2024-11-13T18:27:36,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T18:27:36,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731522456334) roll requested 2024-11-13T18:27:36,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522456342 2024-11-13T18:27:36,349 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:27:36,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,351 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 47817b19762749c530148ffa4f05a124, disabling compactions & flushes 2024-11-13T18:27:36,352 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1731522425912.47817b19762749c530148ffa4f05a124. 2024-11-13T18:27:36,351 WARN [LogRoller {}] wal.FSHLog$SyncRunner(501): interrupted java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Thread.join(Thread.java:1313) ~[?:?] at java.lang.Thread.join(Thread.java:1381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.shutDown(FSHLog.java:499) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.shutDownSyncRunners(FSHLog.java:290) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCleanUpResources(FSHLog.java:283) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.doReplaceWriter(AbstractFSWAL.java:2067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$replaceWriter$7(AbstractFSWAL.java:1017) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.replaceWriter(AbstractFSWAL.java:1016) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1106) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] 2024-11-13T18:27:36,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731522456342 2024-11-13T18:27:36,352 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1731522425912.47817b19762749c530148ffa4f05a124. 2024-11-13T18:27:36,352 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1731522425912.47817b19762749c530148ffa4f05a124. after waiting 0 ms 2024-11-13T18:27:36,353 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1731522425912.47817b19762749c530148ffa4f05a124. 2024-11-13T18:27:36,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T18:27:36,354 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1731522425912.47817b19762749c530148ffa4f05a124. 2024-11-13T18:27:36,354 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1731522425912.47817b19762749c530148ffa4f05a124. 2024-11-13T18:27:36,354 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 47817b19762749c530148ffa4f05a124: Waiting for close lock at 1731522456351Disabling compacts and flushes for region at 1731522456351Disabling writes for close at 1731522456352 (+1 ms)Writing region close event to WAL at 1731522456354 (+2 ms)Closed at 1731522456354 2024-11-13T18:27:36,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): 47817b19762749c530148ffa4f05a124[MyCF] 2024-11-13T18:27:36,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 47817b19762749c530148ffa4f05a124, because it is not online on us 2024-11-13T18:27:36,355 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:27:36,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:27:36,699 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1731522456342) 2024-11-13T18:27:36,700 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/352e0933-581c-0601-68d7-17fadf4e256c 2024-11-13T18:27:37,154 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=524 (was 500) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2299 (was 3123)